/*******************************************************************************
filename      Parser.cpp
author        Craig Williams
last modified 03-10-2010
version       8.9

Description:
  Provides a parser library, as well as several string manipulation funcitons
    to allow for easy parseing of a file. The library handles all file input
    and as much clean up as possiable.

  WARNING: This library heavly uses malloc. The character array returned by
           GrabToken(), PeekToken(), and GrabBytes(<bytes>) must be freeded with
           free(<char *>); manually.

           There are no memory leaks inside the parser library.

  NOTE: Unicode is not supported.

  NOTE: Only Little Endian is supported. This only effects GrabBinary*()

  Parser Functions:
      Parser()        - Initializer. Takes in a string that contains
                          the file to load, as well as how many bytes to read
                          in from the file at one time (Recommended: 1 KB)
     ~Parser()        - Frees all the memory used by the parser
      LoadFile        - Opens and loads a new text file into the parser.
      LoadMemory      - Loads a chunk of memory (char *) into the parser for
                          processing. In other words, don't use internal file
                          handleing.
      LoadMemoryLen   - Same as above; however, it takes in the lenght of the
                          memory.
      Enable          - Enables the specifed Parser feature
      IsEnabled       - Returns !0 if the specified feature(s) are enabled
      Disable         - Disables the specified Parser feature
      AddTokenSet     - Adds another Token set. Allows for easy switching
                          between various parseing modes, as well as a built in
                          branching system.
      SetTokenSet     - Changes the current Token Set.
      GetTokenSet     - Returns the current Token Set.
      AddTokenSeparator - Adds a new "Token Separator" to the current token set.
                            A Token Separator is simply the string to check for
                            in the string we are parseing. Depending on the
                            flags you pass in, you can remove this token
                            from being returned from Grab/Peek Token, or do
                            some basic branching.
      GrabToken       - Gets the next token from the string, and then updates
                          our position in the string.
      PeekToken       - Gets the next token from the string without updateing
                          our position in the string. All token logic, except
                          for callbacks, is taken into account.
      GrabInt         - Gets the next token from the string, and attempts to
                          convert it to a int. This function handles all
                          required cleanup.
      GrabFloat       - Gets the next token from the string, and attempts to
                          convert it to a float. This function handles all
                          required cleanup.
      GrabBinaryInt   - Reads in the next sizeof(int) bytes, and returns them
      GrabBinaryFloat - Reads in the next sizeof(float) bytes, and returns them
      GrabByte        - Reads in the next byte, and returns it.
      PeekByte        - Same as GrabByte(); however, the Parser state is not
                          modified.
      GrabBytes       - Reads in a specific number of bytes from the file, and
                          returns them.
      Seek            - Searches the string for the specified token, and then
                          updates our position in the string, to the character
                          directly following the token.
      SetFilePosition - Changes our location in the file, and reads in the new
                          data, from the specifed location.
      GetFilePosition - Returns the current file position.
      GetFileSize     - Returns the size of the loaded file/block of memory
      End             - Returns 1 if there was a error, or if we reached the end
                          of the file - No more processing possiable.
      GetParams       - Returns the last paramater associated with the last
                          found token separator.
      ErrorCode       - Returns the current error code (check defines in
                          parser_core.h).
      PrintErrorCode  - Prints out the Error Code, in a text format to the
                          command prompt.
      ParserMemoryUsage-Returns a estimate of how much memory the Parser is
                          currently using. This number is in bytes. Typically,
                          this value will be 100% correct; however, it can
                          deviate if a RegEx fails to compile.
      GenericDiscard  - Supplied Parser callback, that discards the next token

  String Manipulaiton functions:
      RemoveWhiteSpaces - Removes all white spaces from a string (Space, New
                            Line, Carriage Return, and Tab)
      ToUpper     - Converts all letters in a string to upper case
      ToLower     - Converts all letters in a string to lower case
      Dup         - Creates a copy of the specfied string
      DupLen      - Same as above, but it takes in the length of the string
      DupRange    - Creates a copy of a specific part of a string
      DupRangeFile- Creates a copy of a specific part of a file
      Cmp         - Compairs two strings. If they are equal, 1 is returned. If
                      not, 0 is returned.

  Private Parser Functions:
      GrabNextChunk   - Loads in the next chunk of the file. The max amount of
                          memory allocated/the buffer size can be calculated
                          from: BufSize + LongestSep - 1 + 2 * sizeof(int)
      RebuildHash     - Rebuilds the Parser hash table, based on the first
                          character of each token separator.
      EnableWildcard  - Converts a Globbing based token separator into it's
                          internal representation.
      DisableWildcard - Converts a Globbing based token separator back to it's
                          original form.
      SortTokenSet    - Sorts all the tokens based on the first character,
                          followed by the length of the string.
      GTChar          - Gets the next token without any features enabled
      GTCharCaseIns   - Case insensitive version of GTChar
      GTCharHash      - Hased version
      GTCharHashCaseIns-Hased & Case insensitive version
      GTCharWild      - Globbing based search algorithm
      ForwardSearch   - Actual globbing logic
      ProcessToken    - Handles the logic of a Separator_str, when ever one is
                          found.
      ProcessTokenWild- ProcessToken, Globbing & RegEx edition
      GrabLeftover    - Retrieves any left over data in SP or from the file when
                          the end of the file is reached.
      PDupRangeFile   - Optimized version of DupRangeFile, that relies on the
                          internal Parser state.
      ReadBinary      - Reads the specified number of bytes from the stream.
      PreserveBufferState - Preserves the current Parsers buffer state
      RestoreBufferState  - Restores the specified buffer state
      PreserveTSetHistory - Preserves the Parser token set history
      RestoreTSetHistory  - Restores the specified token set history

      PrintCompiled   - Prints out the speicifed byte code in a human readable
                          form.
      EnableRegEx     - Compiles a RegEx string to it's byte code form
      DisableRegEx    - Restores the original RegEx string
      ForwardSearchReg- Actual RegEx search function
      GTCharRegEx     - Gets the next token, based on the RegEx's
      HandleEscapes   - Converts all escape sequences into a processable form
      BuildRange      - Converts a "[...]" to a binary range.
      Compile         - Compiles a RegEx to it's byte code
      InvalidRegEx    - Performs some basic error checking on a RegEx
      UpdateThreads   - Processes the actual RegEx byte code
      ShiftRight      - Shifts part of a RegEx byte code over by a specified
                          number of bytes.
      WriteThreadBefore-Writes a THREAD or JUMP op code into the RegEx byte code
*******************************************************************************/
#define _CRT_SECURE_NO_DEPRECATE    /* Disable Microsoft deperication warning */
#define _CRT_SECURE_NO_WARNINGS     /* Disable Microsoft deperication warning */

#include <stdlib.h>                 /* malloc(), realloc(), free, atoi, atof  */
#include <stdio.h>                  /* printf(), fopen, fclose, fseek, fread  */
#include <string.h>                 /* strlen(), memcpy(), memmove()          */
#include <limits.h>                 /* INT_MAX                                */

#include "Parser.h"

#define TSET_HISTORY_DEPTH      10

  /* P->Flags Private Defines                                                 */
#define FPOS_MODIFIED    (1 << 12)  /* P->FPos != fseek(P->FP)                */
#define PARSING_MEM      (1 << 13)  /* Parsing Memory from LoadMemory()       */
#define ALLOC_FLAG       (1 << 14)  /* P->SP must be reallocated (expanded)   */
#define END_OF_FILE      (1 << 15)  /* EOF, but there are still bytes to proc */
#define DONT_FREE_FILE   (1 << 16)
#define DONT_FREE_SP     (1 << 17)

  /* Token Sep Flags                                                          */
#define RETURN_TOKEN     (1 <<  0)  /* Should the token be returned?          */
#define IGNORE_TOKEN     (1 <<  1)  /* Should the token be ignored?           */
#define FREE_TOKEN       (1 <<  2)  /* Can we free the token?                 */

  /* Token Sep Flags - Wildcard Extension                                     */
#define WILD_START       (1 <<  3)  /* "*foo bar " - Wildcard at the start.   */
#define WILD_MIDDLE      (1 <<  4)  /* " foo*bar " - Wildcard in the middle.  */
#define WILD_END         (1 <<  5)  /* " foo bar*" - Wildcard at the end.     */
#define WILD_FLAGS       (WILD_START | WILD_MIDDLE | WILD_END)

  /* Case Insensitive Extension - Helper Defines                              */
#define TOLOWER(c)       (unsigned char)(c >= 'A' && c <= 'Z' ? c + ('a'-'A'):c)
#define TOUPPER(c)       (unsigned char)(c >= 'a' && c <= 'z' ? c - ('a'-'A'):c)
#define FLAG_TOLOWER(c)  if(P->Flags & PARSER_CASE_INSENSITIVE) c = TOLOWER(c);

  /* Ownership helper defines.                                                */
  /* If PARSER_CONST_* or PARSER_OWNS_* is set, the string should not be      *
   *   duplicated. a << 3 is based on the difference between the respective   *
   *   defines.                                                               */
#define SHOULD_DUP(a)    (!(P->Flags & (a | (a << 3))))

  /* Enable() helper defines. PE_REDUNDANT_CHECK() checks to see if any *
   *   flags conflict. PARSER_OWNS_* and PARSER_CONST_* cannot be used        *
   *   together. If both flags are set, PE_REDUNDANT_CHECK() assumes that     *
   *   PARSER_CONST_* was desired, since it does not free the string.         */
#define PE_REDUNDANT_CHECK(f, a) if((f & (a | (a << 3)))==(a|(a<<3)))f&=~(a<<3);

  /* Only used for the return type of ProcessToken*(), to preserve alignment  */
#define BYTE unsigned char


  /* RegEx extension defines                                                  */
#define GET_INT(data, i)        (*(int *)(data + i))
#define GET_INT_INC(data, i)    GET_INT(data, i);{i += sizeof(int);}
#define SET_INT(data, i, j)     {*(int *)(data + i) = j;}
#define SET_INT_INC(data, i, j) {SET_INT(data, i, j) {i += sizeof(int);}}
#define SET_BIT_CHAR(range, c)  {range[c >> 3] |= (BYTE)(1 << ((BYTE)c& 0x07));}

  /* Termination                                                              */
#define PASS               1        /* {PASS}                                 */

  /* Comparisions                                                             */
#define LITERAL            2        /* {LIT, char}                            */
#define RANGE              3        /* {RANGE, char[32]}                      */
#define WILD               4        /* {WILD}                                 */

  /* Assertions                                                               */
#define WORD_BOUNDRY       5        /* {WORD_BOUNDRY}                         */
#define WORD_BOUNDRY_NOT   6        /* {WORD_BOUNDRY_NOT}                     */
#define WORD_START         7        /* {WORD_START}                           */
#define WORD_END           8        /* {WORD_END}                             */
#define LINE_START         9        /* {LINE_START}                           */
#define LINE_END          10        /* {LINE_END}                             */

  /* Logic                                                                    */
#define JUMP              11        /* {JUMP, goto}                           */
#define THREAD            12        /* {THREAD, goto}                         */
#define THREAD_NON_GREEDY 13        /* Same as THREAD                         */

  /* Pops a thread in the RegEx UpdateThreads() function.                     */
#define POP_THREAD {threads[t--] = threads[--nthreads];}

  /* Determins if a character is a RegEx Word character                       */
#define IS_WORD(c) ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || \
                    (c >= '0' && c <= '9') || (c == '_'))

struct  TokenSet_str;
struct Separator_str;
struct    Parser_str;
struct     RegEx_str;
struct    Thread_str;

typedef struct  TokenSet_str  TokenSet_str;
typedef struct Separator_str Separator_str;
typedef struct    Parser_str    Parser_str;
typedef struct     RegEx_str     RegEx_str;
typedef struct    Thread_str    Thread_str;

static void  ParserDeInit       (Parser_str *P);
static long  GetFilePosition    (Parser_str *P);
static int   SetFilePosition    (Parser_str *P, long fpos);
static BYTE  PeekByte           (Parser_str *P, int offset);
static int   SetTokenSet        (Parser_str *P, int tokenset);
static int   End                (Parser_str *P);

static void  GrabNextChunk      (Parser_str *P);
static void  RebuildHash        (Parser_str *P, int tset);
static void  EnableWildcard     (Separator_str *sep);
static void  DisableWildcard    (Separator_str *sep);
static void  SortTokenSet       (Parser_str *P, int tset);
static char *GTChar             (Parser_str *P);
static char *GTCharCaseIns      (Parser_str *P);
static char *GTCharHash         (Parser_str *P);
static char *GTCharHashCaseIns  (Parser_str *P);
static char *GTCharWild         (Parser_str *P, int findstart);
static long  ForwardSearch      (Parser_str *P, int spos, int findstart, Separator_str *sep);
static BYTE *ProcessToken       (Parser_str *P, struct Separator_str *sep, int *spos);
static BYTE *ProcessTokenWild   (Parser_str *P, struct Separator_str *sep, int *spos, long*);
static char *GrabLeftover       (Parser_str *P);
static char *PDupRangeFile      (Parser_str *P, int start, int end);
static int   ReadBinary         (Parser_str *P, void *dest, int size);
static void *PreserveBufferState(Parser_str *P);
static void  RestoreBufferState (Parser_str *P, void *state);
static void *PreserveTSetHistory(Parser_str *P);
static void  RestoreTSetHistory (Parser_str *P, void *state);

  /* RegEx Extension Functions                                                */
/*static void  PrintCompiled      (BYTE *data);*/
static int   EnableRegEx        (Parser_str *P, Separator_str *sep);
static void  DisableRegEx       (Separator_str *sep);
static long  ForwardSearchReg   (Parser_str *P, int   spos, BYTE *data,    BYTE prevchar);
static char *GTCharRegEx        (Parser_str *P);
static BYTE *HandleEscapes      (Parser_str *P, BYTE *buf,  char **escs);
static BYTE *BuildRange         (BYTE *exp,  BYTE  *range, char **escs,int*esc);
static BYTE *Compile            (Parser_str *P, char *exp);
static int   InvalidRegEx       (Parser_str *P, const char *exp);
static int   UpdateThreads      (Parser_str *P, BYTE *buf,  BYTE  *data,  Thread_str *threads,
                                 int  *nt,   int   *procs, int    loop,
                                 int  *greed);
static int   ShiftRight         (BYTE *data, int    start,  int  end,
                                 int   offs, int   *groups, int  grouppos);
static int   WriteThreadBefore  (BYTE *data, int    dpos,   int *groups,
                                 int  *gpos, BYTE   cmd,    int  go_to);

struct Parser_str
{
  TokenSet_str  *TSets ;      /* List of all the token sets                   */
  Separator_str *CurSep;      /* Pointer to the active token set list         */

  unsigned char *SP    ;      /* Pointer to the actual data to parse          */
  long  SPos           ;      /* Where (index) we last stoped parseing, in SP */
  char *File           ;      /* Name/Path of the file to parse               */
  long  FPos           ;      /* Where we are in the actual file              */
  FILE *FP             ;      /* Store the FILE *                             */
  long  CopyFPos       ;      /* Starting position in the file to copy from   */
  int   CurBufSize     ;      /* How many characters from the file are in SP  */
  int   ActualSize     ;      /* Bytes from the file, plus the padding        */
  int   BufSize        ;      /* How many bytes to read in from a file        */
  int   Error          ;      /* Did a error occur?                           */
  int   LongestSep     ;      /* Longest length of the separator tokens       */
  long  Flags          ;      /* Holds all Parser option defines              */
  int   NumOfSep       ;      /* Alias for the above.                         */
  int   CurTokenSet    ;      /* Stores the index of the active token set     */
  int   NumOfTokenSet  ;      /* How many token sets there are                */

    /* Keep track of the token sets that have been switched to. This info is  *
     *   used when the -2 SwitchTo option is used. IE, switch to the last to  *
     *   token set.                                                           *
     * Hard coded deque.                                                      */
  int   TSetHistory[TSET_HISTORY_DEPTH];
  int   TSetHistoryPos ;

  RegEx_str *RegEx;
  void      *LParam;

  Parser *PClass;
};

struct TokenSet_str
{
  unsigned int   *Hash;       /* Hash table. Uses first char of each sep.     */
  Separator_str  *Sep;        /* Array of Token Separators.                   */
  int NumOfSep;               /* Number of Separators in the token set.       */
};

struct Separator_str
{
  unsigned char *Token;       /* Pointer to the actual token                  */
  int   Flags   ;             /* Should the token be returned, or ignored?    */
  int   SwitchTo;             /* Which Token Set to use, after token is found */
  int   Len     ;             /* Stores the length of the token               */
  PCBACK Fun    ;             /* Function to call when the token is found     */
  void *Params  ;             /* Paramaters to pass to the callback           */
};

struct RegEx_str
{
  Thread_str *Threads;        /* Buffer that stores all the RegEx threads     */
  int  NThreads;              /* Max number of supported threads              */
  int *Procs;                 /* Keeps track of what parts of a RegEx executed*/
  int  NProcs;                /* Size of the Largest RegEx                    */
};

struct Thread_str
{
  int DPos;                   /* Where in the RegEx to execute                */
  int Greed;                  /* How greedy a non-greedy repetition char is   */
};

/*******************************************************************************
   Function: ParserInit()

Description: Constructor - Creates our initial Token Set, and sets up some
              default values.

     Inputs: file    - String to the file to load in
             bufsize - How many bytes to read in from the file at a time.

    Outputs: N/A
*******************************************************************************/
Parser::Parser(const char *file, int bufsize)
{
  P = (Parser_str *)malloc(sizeof(Parser_str));

  if(!P){return;}

  if(bufsize <= 0)
    bufsize = 1024;

  P->PClass          =  this   ;
  P->CurSep          =  0      ; /* Alias to the Current Separator_str        */
  P->SP              =  0      ; /* The current chunk of data we are parseing */
  P->SPos            =  0      ; /* Our actual position in P->SP              */
  P->File            =  Dup(file);/*P->File to open and parse                 */
  P->FPos            =  0      ; /* Our position in the actual file           */
  P->FP              =  0      ; /* No file is open.                          */
  P->ActualSize      =  0      ; /* How many bytes are actually in the buffer */
  P->CurBufSize      =  0      ; /* Empty buffer                              */
  P->BufSize         =  bufsize; /* Save the requested buffer size            */
  P->Error           =  0      ; /* No Errors currently exist!                */
  P->NumOfSep        =  0      ; /* No Separator_str exist                    */
  P->CurTokenSet     = -1      ; /* No active Token set is selected           */
  P->NumOfTokenSet   =  0      ; /* No Token Sets exits                       */
  P->LongestSep      =  0      ; /* No seperators, so 0 is the max length     */
  P->CopyFPos        = -1      ; /* -1 siginifes no file fragmentation        */
  P->Flags           =  0      ; /* No options are enabled by default         */
  P->TSetHistoryPos  =  0      ;
  memset(P->TSetHistory,0, sizeof(P->TSetHistory));
  P->RegEx           =  0      ;
  P->LParam          =  0      ;

    /* Create the initial empty token set */
  P->TSets = (TokenSet_str*)malloc(sizeof(TokenSet_str));
  if(!P->TSets){P->Error = PARSER_OUT_OF_MEMORY;return;}

  P->TSets[0].Hash     = 0;
  P->TSets[0].Sep      = 0;
  P->TSets[0].NumOfSep = 0;

    /* We've created our initial token set, so update P to siginify that we   *
     *   have done as such.                                                   */
  P->CurTokenSet       =  0;
  P->NumOfTokenSet     =  1;
}

Parser::Parser(const Parser &r)
{
  P = 0;
  *this = r;
}

/*******************************************************************************
   Function: ParserDeInit()

Description: Destructor - Frees all the data that Parser grabbed.

     Inputs: N/A

    Outputs: N/A
*******************************************************************************/
Parser::~Parser(void)
{
  ParserDeInit(P);
  P = 0;
}

void ParserDeInit(Parser_str *P)
{
  int i;
  int j;

  if(!P)
    return;

  if(!(P->Flags & DONT_FREE_SP))
    if(P->SP)
      free(P->SP);

  for(i = 0; i < P->NumOfTokenSet; ++i)
  {
    if(P->TSets[i].Sep)
    {
      for(j = 0; j < P->TSets[i].NumOfSep; ++j)
        if(P->TSets[i].Sep[j].Flags & FREE_TOKEN)
          free(P->TSets[i].Sep[j].Token);

      free(P->TSets[i].Sep);
    }

    if(P->TSets[i].Hash)
      free(P->TSets[i].Hash);
  }

  if(P->TSets)
    free(P->TSets);

  if(P->RegEx)
  {
    if(P->RegEx->Threads)
      free(P->RegEx->Threads);
    if(P->RegEx->Procs)
      free(P->RegEx->Procs);
    free(P->RegEx);
  }

  if(!(P->Flags & DONT_FREE_FILE))
    if(P->File)
      free(P->File);

  if(P->FP)
    fclose(P->FP);

  free(P);
}

/*******************************************************************************
   Function: operator=

Description: Destroies the current state, and duplicates the specified Parser's
               state.

     Inputs: _r - Parser to duplicate

    Outputs: Reference to this Parser
*******************************************************************************/
Parser &Parser::operator=(const Parser &_r)
{
  Parser &r((Parser &)_r);
  long options;
  int tset;
  int sep;
  ParserDeInit(P);
  if(!r.P) {P = 0; return *this;}

  P = (Parser_str *)malloc(sizeof(Parser_str));

  if(!P){return *this;}

  memset(P, 0, sizeof(*P));

  P->PClass         =  this;
  P->BufSize        =  r.P->BufSize;
  P->TSetHistoryPos =  0;
  P->LParam         =  0;
  memset(P->TSetHistory,0, sizeof(P->TSetHistory));

    /* Create the initial empty token set */
  P->TSets = (TokenSet_str*)malloc(sizeof(TokenSet_str));
  if(!P->TSets){P->Error = PARSER_OUT_OF_MEMORY;return *this;}

  P->TSets[0].Hash     = 0;
  P->TSets[0].Sep      = 0;
  P->TSets[0].NumOfSep = 0;

    /* We've created our initial token set, so update P to siginify that we   *
     *   have done as such.                                                   */
  P->CurTokenSet       =  0;
  P->NumOfTokenSet     =  1;

  options = r.P->Flags & PARSER_ALL_OPTIONS;

    /* Disable any options in r. This must be done to convert any wildcards   *
     *   or RegEx tokens back to their original form. From here, we can then  *
     *   directly load all the tokens into this parser, via AddTokenSeparator */
  r.Disable(options);

  for(tset = 0; tset < r.P->NumOfTokenSet; ++tset)
  {
    if(tset)
      AddTokenSet();
    for(sep = 0; sep < r.P->TSets[tset].NumOfSep; ++sep)
      AddTokenSeparator((const char *)r.P->TSets[tset].Sep[sep].Token,
                        r.P->TSets[tset].Sep[sep].Flags & RETURN_TOKEN,
                        r.P->TSets[tset].Sep[sep].SwitchTo,
                        r.P->TSets[tset].Sep[sep].Flags & IGNORE_TOKEN,
                        r.P->TSets[tset].Sep[sep].Fun,
                        r.P->TSets[tset].Sep[sep].Params);
  }

    /* Load in the file/memory r was working on */
  if(r.P->Flags & PARSING_MEM)
    LoadMemoryLen((const char *)r.P->SP, r.P->ActualSize);
  else
    LoadFile(r.P->File);

  SetFilePosition(r.GetFilePosition());

  SetTokenSet(r.GetTokenSet());

  P->TSetHistoryPos =  r.P->TSetHistoryPos;
  memcpy(P->TSetHistory, r.P->TSetHistory, sizeof(P->TSetHistory));

  Enable(options);
  r.Enable(options);

  return *this;
}

/*******************************************************************************
   Function: LoadFile()

Description: Loads a new file into the parser.

     Inputs: *file - Name of the file to open and parse.

    Outputs: 1 - File sucessfully loaded!
             0 - Error occured.
*******************************************************************************/
int Parser::LoadFile(const char *file)
{
  if(!P)
    return 0;

  if(!(P->Flags & DONT_FREE_FILE))
    if(P->File)
      free(P->File);

  if(P->Flags & PARSING_MEM && P->SP)
  {
    if(!(P->Flags & DONT_FREE_SP))
      free(P->SP);
    P->SP = 0;
  }
  else
  if(P->FP)
  {
    fclose(P->FP);
    P->FP = 0;
  }

  P->SPos       =  0;

  if(SHOULD_DUP(PARSER_CONST_FILE_NAME))
    P->File     = Dup(file);
  else
    P->File     = (char *)file;

  P->FPos       =  0;
  P->CopyFPos   = -1;
  P->CurBufSize =  0;
  P->ActualSize =  0;
  P->Flags     &= ~(END_OF_FILE | PARSING_MEM | DONT_FREE_FILE | DONT_FREE_SP);
  P->Error      =  0;

  if(P->Flags & PARSER_CONST_FILE_NAME) P->Flags |= DONT_FREE_FILE;

  return 1;
}

/*******************************************************************************
   Function: LoadMemory()

Description: Loads a chunk of memory into the parser library, instead of useing
               the built in file handleing. Note: memory is coppied into P->SP.
               You should free memory afterwards, if you don't need it.

     Inputs: memory - Pointer to the memory to process

    Outputs: 1 - Memory looks valid.
             0 - Memory is not valid.
*******************************************************************************/
int Parser::LoadMemory(const char *memory)
{
  if(!memory)
    return 0;
  return LoadMemoryLen(memory, (int)strlen(memory));
}

/*******************************************************************************
   Function: LoadMemoryLen()

Description: Loads a chunk of memory into the parser library, instead of useing
               the built in file handleing. Note: memory is coppied into P->SP.
               You should free memory afterwards, if you don't need it.

     Inputs: mem - Pointer to the memory to process
             len - Length (bytes) of the memory

    Outputs: 1 - Memory looks valid.
             0 - Memory is not valid.
*******************************************************************************/
int Parser::LoadMemoryLen(const char *mem, int len)
{
  if(!P)
    return 0;

  if(!mem || !len)
  {
    P->Error = PARSER_MEMORY_NOT_VALID;
    return 0;
  }

  if(P->Flags & PARSING_MEM)
  {
    if(!(P->Flags & DONT_FREE_SP))
      if(P->SP)
        free(P->SP);
  }
  else
  if(P->SP)
    free(P->SP);

  if(P->FP)
  {
    fclose(P->FP);
    P->FP = 0;
  }

  if(!(P->Flags & DONT_FREE_FILE))
    if(P->File)
      free(P->File);

  if(SHOULD_DUP(PARSER_CONST_LOAD_MEMORY))
    P->SP = (unsigned char *)DupRange(mem, 0, len);
  else
    P->SP = (unsigned char *)mem;

  P->CurBufSize = len;
  P->ActualSize = len;
  P->File       =  0;
  P->FPos       = len;
  P->CopyFPos   = -1;
  P->SPos       =  0;
  P->Error      =  0;
  P->Flags     |= END_OF_FILE | PARSING_MEM;

  if(!(P->Flags & PARSER_CONST_LOAD_MEMORY))
    P->Flags &= ~DONT_FREE_SP;
  else
    P->Flags |= DONT_FREE_SP;

  return 1;
}

/*******************************************************************************
   Function: Enable()

Description: Enables a varitey of options that directly modify the behavior or
               performance of the Parser.

     Inputs: flags - PARSER_* options to enable.

    Outputs: N/A
*******************************************************************************/
void Parser::Enable(int flags)
{
  int tset;
  int sep;
  int disable = 0;
  int hash;
  int sort;

  if(!P)
    return;

    /* Remove any flags that do not have a external define set to them.       */
  flags &= PARSER_ALL_OPTIONS;

    /* Remove any redundtant flags. IE, PARSER_CONST_* and PARSER_OWN_*       *
     *   conflict. As such, we must disable PARSER_OWN_*, since it's safer.   */
  PE_REDUNDANT_CHECK(flags, PARSER_CONST_FILE_NAME);
  PE_REDUNDANT_CHECK(flags, PARSER_CONST_LOAD_MEMORY);
  PE_REDUNDANT_CHECK(flags, PARSER_CONST_TOKEN_SEPS);

    /* Remove any flags that are already set.                                 */
  flags -= (P->Flags & flags) & PARSER_ALL_OPTIONS;

    /* If RegEx, disable conflicting flags */
  if(flags & PARSER_REGULAR_EXPRESSIONS)
  {
    disable |= P->Flags & (PARSER_HASH |PARSER_SORT_TOKEN_SEPS|PARSER_WILDCARD);
    flags   &= ~(PARSER_HASH | PARSER_SORT_TOKEN_SEPS | PARSER_WILDCARD);
  }

  if(P->Flags & PARSER_REGULAR_EXPRESSIONS)
    flags   &= ~(PARSER_HASH | PARSER_SORT_TOKEN_SEPS | PARSER_WILDCARD);

  hash = flags & PARSER_HASH;
  sort = flags & PARSER_SORT_TOKEN_SEPS;

  if(flags & PARSER_OWNS_FILE_NAME)   disable |= PARSER_CONST_FILE_NAME;
  if(flags & PARSER_OWNS_LOAD_MEMORY) disable |= PARSER_CONST_LOAD_MEMORY;
  if(flags & PARSER_OWNS_TOKEN_SEPS)  disable |= PARSER_CONST_TOKEN_SEPS;

    /* PARSER_CASE_INSENSITIVE is mainly handled in GrabToken(). The token    *
     *   separators are not modifed, since PARSER_CASE_INSENSITIVE can be     *
     *   disabled, and the original state of the token seps must be preserved.*
     *   As such, a extra copy of the string would have to be stored, thus    *
     *   increaseing memory.                                                  *
     *                                                                        *
     * This flag effects:                                                     *
     *   PARSER_HASH            - Since the first character of each token is  *
     *                               hashed, we have to rebuild the hash since*
     *                               the first character may have changed.    *
     *   PARSER_SORT_TOKEN_SEPS - Sorting is based on the first character,    *
     *                              followed by the token len. The tokens must*
     *                              be re sorted, since the first character   *
     *                              may have changed.                         */
  if(flags & PARSER_CASE_INSENSITIVE)
  {
    if(P->Flags & PARSER_HASH)            hash = 1;
    if(P->Flags & PARSER_SORT_TOKEN_SEPS) sort = 1;
  }

    /* This option directly modifies the token separators for performance     *
     *   and simplicity reasons. See EnableWildcard() for more information    *
     *   on this modification. Since the token seps are modified, the hash    *
     *   and sorting may have to be recalculated.                             */
  if(flags & PARSER_WILDCARD)
  {
    for(tset = 0; tset < P->NumOfTokenSet; ++tset)
      for(sep = 0; sep < P->TSets[tset].NumOfSep; ++sep)
        EnableWildcard(P->TSets[tset].Sep + sep);

    if(P->Flags & PARSER_HASH)            hash = 1;
    if(P->Flags & PARSER_SORT_TOKEN_SEPS) sort = 1;
  }

  if(disable)
    Disable(disable);

  if(flags & PARSER_REGULAR_EXPRESSIONS)
  {
    P->RegEx = (RegEx_str *)malloc(sizeof(RegEx_str));
    P->RegEx->Threads  = 0;
    P->RegEx->NThreads = 0;
    P->RegEx->Procs    = 0;
    P->RegEx->NProcs   = 0;
    for(tset = 0; tset < P->NumOfTokenSet; ++tset)
      for(sep = 0; sep < P->TSets[tset].NumOfSep; ++sep)
        if(EnableRegEx(P, P->TSets[tset].Sep + sep))
        {
          P->TSets[tset].Sep[sep] = P->TSets[tset].Sep[--P->TSets[tset].NumOfSep];
          --sep;
        }
  }

  P->Flags |= flags;

  if(hash)
    for(tset = 0; tset < P->NumOfTokenSet; ++tset)
      RebuildHash(P, tset);
  else
  if(sort)
    for(tset = 0; tset < P->NumOfTokenSet; ++tset)
      SortTokenSet(P, tset);
}

/*******************************************************************************
   Function: IsEnabled()

Description: Checks to see if the specified options are enabled. If all the
               options are enabled, a 1 will be returned. Else, 0 will be
               returned.

     Inputs: flags - Options to check if they are enabled.

    Outputs: 1 - All the specified flags are enabled.
             0 - One or more flags are not enabled.
*******************************************************************************/
int Parser::IsEnabled(int flags) const
{
  if(!P)
    return 0;
  return (P->Flags & flags) == flags;
}

/*******************************************************************************
   Function: Disable()

Description: Disables the specified options if they are enabled.

     Inputs: flags - Options to disable.

    Outputs: N/A
*******************************************************************************/
void Parser::Disable(int flags)
{
  int tset;
  int sep;

  if(!P)
    return;

    /* Remove any non publicly defined options, and then only disable the     *
     *   options that are actually enabled.                                   */
  flags &= P->Flags & PARSER_ALL_OPTIONS;

  if(flags & PARSER_HASH)
  {
    for(tset = 0; tset < P->NumOfTokenSet; ++tset)
    {
      free(P->TSets[tset].Hash);
      P->TSets[tset].Hash = 0;
    }
      /* This flag is removed here, to prevent PARSER_WILDCARD and            *
       *   PARSER_CASE_INSENSITIVE from preformaing additional operations.    */
    P->Flags &= ~PARSER_HASH;
  }

    /* PARSER_WILDCARD directly modifies the token seps. As such, we have to  *
     *   restore the original tokens. See DisableWildcard() for information   *
     *   about this modification.                                             *
     *                                                                        *
     * This option effects:                                                   *
     *   PARSER_HASH & PARSER_SORT_TOKEN_SEPS since the token seps are        *
     *     directly modified.                                                 */
  if(flags & PARSER_WILDCARD)
    for(tset = 0; tset < P->NumOfTokenSet; ++tset)
    {
      for(sep = 0; sep < P->TSets[tset].NumOfSep; ++sep)
        DisableWildcard(P->TSets[tset].Sep + sep);

      if(P->Flags & PARSER_HASH)
        RebuildHash(P, tset);
      else
      if(P->Flags & PARSER_SORT_TOKEN_SEPS)
        SortTokenSet(P, tset);
    }

    /* Disable case insensitivity                                             */
  if(flags & PARSER_CASE_INSENSITIVE)
  {
    P->Flags &= ~PARSER_CASE_INSENSITIVE;
    if(P->Flags & PARSER_HASH)
      for(tset = 0; tset < P->NumOfTokenSet; ++tset)
        RebuildHash(P, tset);
      else
      if(P->Flags & PARSER_SORT_TOKEN_SEPS)
        for(tset = 0; tset < P->NumOfTokenSet; ++tset)
          SortTokenSet(P, tset);
  }

  if(flags & PARSER_REGULAR_EXPRESSIONS)
  {
    if(P->RegEx)
    {
      if(P->RegEx->Threads)
        free(P->RegEx->Threads);
      if(P->RegEx->Procs)
        free(P->RegEx->Procs);
      free(P->RegEx);
    }
    P->RegEx = 0;
    for(tset = 0; tset < P->NumOfTokenSet; ++tset)
      for(sep = 0; sep < P->TSets[tset].NumOfSep; ++sep)
        DisableRegEx(P->TSets[tset].Sep + sep);
  }


  P->Flags &= ~flags;
}

/*******************************************************************************
   Function: AddTokenSet()

Description: Adds a new Token Set for the GrabToken function to use

     Inputs: N/A

    Outputs: -1 - Error occured
             0+ - Index of the new Token Set
*******************************************************************************/
int Parser::AddTokenSet(void)
{
  if(!P)
    return -1;

  P->TSets = (TokenSet_str *)
       realloc(P->TSets, sizeof(TokenSet_str) * (P->NumOfTokenSet + 1));

  if(!P->TSets){P->Error = PARSER_OUT_OF_MEMORY;return -1;}

  P->TSets[P->NumOfTokenSet].Sep      = 0;
  P->TSets[P->NumOfTokenSet].NumOfSep = 0;
  P->TSets[P->NumOfTokenSet].Hash     = 0;

  ++P->NumOfTokenSet;

  SetTokenSet(P->NumOfTokenSet - 1);

  return P->NumOfTokenSet - 1;
}

/*******************************************************************************
   Function: SetTokenSet()

Description: Changes the current token set used with GrabToken & such

     Inputs: TokenSet - Index of the desiered Token Set

    Outputs: -1 - Invalid Token Set
             0+ - Current Token Set being used
*******************************************************************************/
int Parser::SetTokenSet(int tokenset)
{
  return ::SetTokenSet(P, tokenset);
}

int SetTokenSet(Parser_str *P, int tokenset)
{
  if(!P || tokenset >= P->NumOfTokenSet || tokenset == -1 || tokenset < -2)
    return -1;

  if(tokenset == -2) /* Switch To the last Token Set                          */
  {
    if(--P->TSetHistoryPos < 0)
      P->TSetHistoryPos = TSET_HISTORY_DEPTH - 1;
    tokenset = P->TSetHistory[P->TSetHistoryPos];
  }
  else
  {
    P->TSetHistory[P->TSetHistoryPos] = P->CurTokenSet;
    if(++P->TSetHistoryPos >= TSET_HISTORY_DEPTH)
      P->TSetHistoryPos = 0;
  }

  P->CurTokenSet = tokenset;
  P->CurSep      = P->TSets[tokenset].Sep;
  P->NumOfSep    = P->TSets[tokenset].NumOfSep;

  return tokenset;
}

/*******************************************************************************
   Function: GetTokenSet()

Description: Returns the current token set.

     Inputs: N/A

    Outputs: -1 - Error Occured
             0+ - Current TokenSet
*******************************************************************************/
int Parser::GetTokenSet(void) const
{
  if(!P)
    return -1;
  return P->CurTokenSet;
}

/*******************************************************************************
   Function: AddTokenSeparator()

Description: Adds a new character to the current array of token separators.

     Inputs: sp       - Pointer to the string to consider as a separator
             Return   - Should the separator be returned by Grab/PeekToken()?
                        This is useful for filtering out specific strings.
             switchto - Automatically switch to the specifed token set, when the
                        token is found.
             ignore   - If this Token is found, just keep going. Orignally
                        designed to be used with strings. For example,
                        \" should be ignored; however, " will be picked out
                        if we don't ignore \"
             fun      - Callback function to call when ever the token is found.
             params   - Paramaters to pass to the callback

    Outputs: 0  - Token added
             1+ - Specifed token is not valid
*******************************************************************************/
int Parser::AddTokenSeparator(const char *sp, int Return, int switchto, int ign,
                              PCBACK fun, void *params)
{
  unsigned int error;
  unsigned int size;
  int len;
  Separator_str *sep;

  if(!P || !sp || !sp[0])
    return PARSER_TOKEN_NULL;

  P->NumOfSep = ++P->TSets[P->CurTokenSet].NumOfSep;

  size = sizeof(Separator_str) * P->NumOfSep;

  if(P->TSets[P->CurTokenSet].Sep == 0)
    P->TSets[P->CurTokenSet].Sep = (Separator_str *)malloc(size);
  else
    P->TSets[P->CurTokenSet].Sep =
      (Separator_str *)realloc(P->TSets[P->CurTokenSet].Sep, size);

  if(!P->TSets[P->CurTokenSet].Sep)
  {
    P->Error = PARSER_OUT_OF_MEMORY;
    return PARSER_OUT_OF_MEMORY;
  }

  P->CurSep = P->TSets[P->CurTokenSet].Sep;

  len = (int)strlen(sp);

  sep = &P->CurSep[P->NumOfSep - 1];

  if(SHOULD_DUP(PARSER_CONST_TOKEN_SEPS))
    sep->Token  = (unsigned char *)DupLen(sp, len);
  else
    sep->Token  = (unsigned char *)sp;

  if(!sep->Token)
  {
    P->Error = PARSER_OUT_OF_MEMORY;
    return PARSER_OUT_OF_MEMORY;
  }

  sep->SwitchTo = switchto;
  sep->Len      = len;
  sep->Fun      = fun;
  sep->Params   = params;
  sep->Flags    = 0;

  if(Return)                                sep->Flags |= RETURN_TOKEN;
  if(ign)                                   sep->Flags |= IGNORE_TOKEN;
  if(!(P->Flags & PARSER_CONST_TOKEN_SEPS)) sep->Flags |= FREE_TOKEN;

  if(len > P->LongestSep)
  {
    P->LongestSep = len;
    P->Flags     |= ALLOC_FLAG;
  }

  if(P->Flags & PARSER_REGULAR_EXPRESSIONS)
  {
    error = EnableRegEx(P, sep);
    if(error)
    {
      --P->TSets[P->CurTokenSet].NumOfSep;
      return error;
    }
  }
  else
  {
    if(P->Flags & PARSER_WILDCARD)
      EnableWildcard(sep);

    if(P->Flags & PARSER_HASH)
      RebuildHash(P, P->CurTokenSet);
    else
    if(P->Flags & PARSER_SORT_TOKEN_SEPS)
      SortTokenSet(P, P->CurTokenSet);
  }
  return 0;
}

/*******************************************************************************
   Function: GrabToken()

Description: Grabs the next token in the character array. This function calls
               a specific helper function, based on the currently enabled
               settings. All of the options were not combined into one function,
               due to performance reasons, with the exception of GTCharWild().
               See GtCharWild() for more information.

     Inputs: N/A

    Outputs: 0 - no valid Token Separators are present
             Pointer to a new character array containing the token
*******************************************************************************/
char *Parser::GrabToken(void)
{
  if(!P || P->Error || P->NumOfSep <= 0)
    return 0;

  P->LParam = 0;

  switch(P->Flags & (PARSER_HASH | PARSER_CASE_INSENSITIVE | PARSER_WILDCARD |
                     PARSER_REGULAR_EXPRESSIONS))
  {
    case  0: return GTChar(P);
    case  1: return GTCharHash(P);
    case  2: return GTCharCaseIns(P);
    case  3: return GTCharHashCaseIns(P);
    case  4:
    case  5:
    case  6:
    case  7: return GTCharWild(P, 0);
    case  8:
    case  9:
    case 10:
    case 11:
    case 12:
    case 13:
    case 14:
    case 15: return GTCharRegEx(P);
    default: return 0;
  }
}

/*******************************************************************************
   Function: PeekToken()

Description: Grabs the next token in the character array, while preserveing
               our current position in the string. All logic except for
               callbacks are taken into account.

             TODO: Disable Callbacks (add a flag)

     Inputs: N/A

    Outputs: Char * to the next token
*******************************************************************************/
char *Parser::PeekToken(void)
{
  void *state;
  void *tsets;
  char *buffer;

  if(!P)
    return 0;

  state  = PreserveBufferState(P);
  tsets  = PreserveTSetHistory(P);
  buffer = GrabToken();

  RestoreBufferState(P, state);
  RestoreTSetHistory(P, tsets);

  return buffer;
}

/*******************************************************************************
   Function: GrabInt()

Description: Grabs the next token in the character array, convert it to a int
               and then return the new int

     Inputs: N/A

    Outputs: The next token converted to a int
*******************************************************************************/
int Parser::GrabInt(void)
{
  int rint;
  char *token;

  if(!P)
    return 0;

  if((token = GrabToken()) == 0)
    return 0;

  rint = atoi(token);

  free(token);

  return rint;
}

/*******************************************************************************
   Function: GrabFloat()

Description: Grabs the next token in the character array, convert it to a float
              and then returns the new float

     Inputs: N/A

    Outputs: The next token converted to a float
*******************************************************************************/
float Parser::GrabFloat(void)
{
  float rfloat;
  char *token;

  if(!P)
    return 0;

  if((token = GrabToken()) == 0)
    return 0.0f;

  rfloat = (float)atof(token);

  free(token);

  return rfloat;
}

/*******************************************************************************
   Function: GrabBinaryInt()

Description: Grabs the next sizeof(int), and convert it to a int

     Inputs: N/A

    Outputs: the next sizeof(int), as a int
*******************************************************************************/
int Parser::GrabBinaryInt(void)
{
  int r = 0;
  ReadBinary(P, &r, sizeof(int));
  return r;
}

/*******************************************************************************
   Function: GrabBinaryFloat()

Description: Grabs the next sizeof(float) bytes, and return it as a float

     Inputs: N/A

    Outputs: The next sizeof(float) as a float
*******************************************************************************/
float Parser::GrabBinaryFloat(void)
{
  float r = 0.0f;
  ReadBinary(P, &r, sizeof(float));
  return r;
}

/*******************************************************************************
   Function: GrabByte()

Description: Grabs the next byte from the string.

     Inputs: N/A

    Outputs: The next byte from the string
*******************************************************************************/
unsigned char Parser::GrabByte(void)
{
  unsigned char ret;

  if(!P)
    return 0;

  if(P->SPos + sizeof(char) > (unsigned int)P->ActualSize)
    SetFilePosition(GetFilePosition());

  if(1 > P->ActualSize)
  {
    P->SPos   = P->ActualSize;
    P->Error  = PARSER_END_OF_FILE;
    return 0;
  }

  ret = P->SP[P->SPos];

  P->SPos += sizeof(char);

  return ret;
}

/*******************************************************************************
   Function: PeekByte()

Description: Peeks at the next byte from the string.

     Inputs: N/A

    Outputs: The next byte from the string
*******************************************************************************/
unsigned char Parser::PeekByte(int offset)
{
  return ::PeekByte(P, offset);
}

unsigned char PeekByte(Parser_str *P, int offset)
{
  int fpos;
  unsigned char ret;

  if(!P)
    return 0;

  fpos = P->FPos - P->ActualSize + P->SPos + offset;

    /* Requesting byte before the start of the file */
  if(fpos < 0)
    return 0;

  if(fpos >= P->FPos || P->SPos + offset < 0)
  {
    if(!P->File)
      return 0;

      /* Outside buffer in memory. Read in the byte */
    if(!P->FP)
      P->FP = fopen(P->File, "rb");
    fseek(P->FP, fpos, SEEK_SET);
    if(!fread(&ret, 1, 1, P->FP))
      ret = 0;
    if(P->Flags & PARSER_CLOSE_FILE)
    {
      fclose(P->FP);
      P->FP = 0;
    }
    else
      fseek(P->FP, P->FPos, SEEK_SET);
    return ret;
  }

  return P->SP[P->SPos + offset];
}

/*******************************************************************************
   Function: GrabBytes()

Description: Grabs the specifed number of bytes from the file.

     Inputs: bytes - How many bytes to read in from the file

    Outputs: The specifed number of bytes
*******************************************************************************/
unsigned char *Parser::GrabBytes(int bytes)
{
  unsigned char *r;

  if(!P || P->Error || bytes <= 0)
    return 0;

  r = (unsigned char *)malloc((unsigned int)bytes + 1);

  if(!r) {P->Error = PARSER_OUT_OF_MEMORY; return 0;}

  if(!ReadBinary(P, r, bytes))
  {
    free(r);
    return 0;
  }

  return r;
}

/*******************************************************************************
   Function: Seek()

Description: Search through the array, until we find the specifed Token, and
               then update our position in the character array

             TODO: Clean me up...

     Inputs: *serach - Token to serach for in the character array

    Outputs: 0 - The specified Token could not be found. Pos was not updated.
             1 - Token found. Updated position to character after the token
*******************************************************************************/
int Parser::Seek(const char *_search)
{
  unsigned char *search = (unsigned char *)_search;
  unsigned char t;
  unsigned char b;
  int   spos;
  int   seekpos = 0;
  int   i, j;
  void *state;

  if(!P || !search || !search[0])
    return 0;

  state = PreserveBufferState(P);

  t = search[seekpos]; FLAG_TOLOWER(t);
  while(!P->Error)
  {
    for(spos = P->SPos; spos < P->ActualSize; ++spos)
    {
      b = P->SP[spos]; FLAG_TOLOWER(b);

      if(t != b)
      {
          /* Due to the buffer size constrant, the part of P->SP that matched *
           *   part of search may no longer be in memory. In order to         *
           *   correctly handle this, we compare search against search and b. *
           *                                                                  *
           *   Memory: "aaabcd"                                               *
           *     Seek: "aabcd"                                                *
           *                                                                  *
           *  In the above, if the buffer size is 1, then P->SP will be       *
           *    1: 'a'   2: 'a'   3: 'a'   4: 'b'   5: 'c'   6: 'd'           *
           *                                                                  *
           *  So, "aa" will match, and then b != a (buf 3). If we set seekpos *
           *    to 0, we will not find the seek value. So, we would scan from *
           *    Memory + 1 to find seek. Since Memory + 1 is no longer in     *
           *    memory, and we know that the first 2 characters matched Seek, *
           *    we can assume this                                            *
           *                                                                  *
           *  Memory: "aaa"                                                   *
           *    The first two 'a' are from seek, and the last one is from b.  *
           *    We do not have to compare against the first a, since we       *
           *    already know it does not match. we can start from search + 1. *
           *                                                                  *
           *    aaa                                                           *
           *     aabcd                                                        *
           *                                                                  *
           *    The above matches, so we set seekpos to 2 ('b').              *
           *    If it did not match, we would compare against Search + 2, and *
           *    so on.                                                        */
        for(i = 1; i <= seekpos; ++i)
        {
          for(j = 0; j < seekpos - i; ++j)
            if(search[j] != search[i + j])
              break;

          if(j == seekpos - i)
          {
            t = search[j]; FLAG_TOLOWER(t);
            if(t == b)
            {
              seekpos = j + 1;
              i = 0;
              break;
            }
          }
        }

        if(i > seekpos)
          seekpos = 0;
      }
      else
        ++seekpos;

      t = search[seekpos];
      if(!t)
      {
        P->SPos     = spos + 1;
        P->CopyFPos = -1;
        free(state);
        return 1;
      }

      FLAG_TOLOWER(t);
    }

    if(!P->File)
    {
      RestoreBufferState(P, state);
      return 0;
    }

    P->SPos = spos;
    GrabNextChunk(P);
  }

  RestoreBufferState(P, state);
  return 0;
}

/*******************************************************************************
   Function: SetFilePosition()

Description: Sets the current file position (in bytes), and then attempts to
               read in the data.

     Inputs: fpos - Requested poisition in the file.

    Outputs: 1 - File position updated, and no errors were found
             0 - Error occured, or you requested a bad file position.
*******************************************************************************/
int Parser::SetFilePosition(long fpos)
{
  return ::SetFilePosition(P, fpos);
}

int SetFilePosition(Parser_str *P, long fpos)
{
  if(!P || fpos < 0)
    return 0;

  if(fpos >= P->FPos - P->ActualSize &&
     fpos <  P->FPos - P->ActualSize + P->CurBufSize)
  {
    if(P->Error == PARSER_END_OF_FILE)
    {
      P->Error  = 0;
      P->Flags |= END_OF_FILE;
    }
      /* If fpos is already inside P->SP, just set our position to that loc   */
    P->SPos = fpos - P->FPos + P->ActualSize;
  }
  else
  {
    P->Error      =  0  ;
    P->Flags     &= ~END_OF_FILE;

      /* fpos is outside the buffer. We must read from the file.              */
    P->FPos       = fpos;
    P->SPos       =  0  ;
    P->Flags     |= FPOS_MODIFIED;
    P->CurBufSize =  0  ;
    P->ActualSize =  0  ;
    P->CopyFPos   = -1  ;

    GrabNextChunk(P);
  }

  return !End(P);
}

/*******************************************************************************
   Function: GetFilePosition()

Description: Gets the current file position

     Inputs: N/A

    Outputs: Current position in the file
*******************************************************************************/
long Parser::GetFilePosition(void) const
{
  return ::GetFilePosition(P);
}

long GetFilePosition(Parser_str *P)
{
  if(!P)
    return 0;

  return P->FPos + P->SPos - P->ActualSize;
}

/*******************************************************************************
   Function: GetFileSize()

Description: Gets the size of the current file/memory block in bytes

     Inputs: N/A

    Outputs: Size of the current file/memory block
*******************************************************************************/
long Parser::GetFileSize(void) const
{
  long  ret;
  FILE *fp;

  if(!P)
    return 0;

  if(!P->File)
    return P->ActualSize; /* Loaded a block of memory */

    /* Loaded a file */
  if(P->FP)
  {
    ((Parser_str *)P)->Flags |= FPOS_MODIFIED;
    fseek(((Parser_str *)P)->FP, 0, SEEK_END);
    return ftell(((Parser_str *)P)->FP);
  }

  fp = fopen(P->File, "rb");

  if(!fp)
    return 0;

  fseek(fp, 0, SEEK_END);
  ret = ftell(fp);
  fclose(fp);

  return ret;
}


/*******************************************************************************
   Function: End()

Description: Returns 1 if we are at the end of the file

     Inputs: N/A

    Outputs: 1 - We reached the end of the file
             0 - We still have data to process
*******************************************************************************/
int Parser::End(void) const
{
  return ::End(P);
}

int End(Parser_str *P)
{
  if(!P || P->Error || !P->SP)
    return 1;

  return 0;
}

/*******************************************************************************
   Function: End()

Description: Returns the last paramater variable associated with the last found
               token.

     Inputs: N/A

    Outputs: Last found token's paramater.
*******************************************************************************/
void *Parser::GetParams(void) const
{
  if(!P)
    return 0;

  return P->LParam;
}

/*******************************************************************************
   Function: ErrorCode()

Description: Grabs the specifed number of bytes from the file

     Inputs: N/A

    Outputs: The specifed number of bytes
*******************************************************************************/
int Parser::ErrorCode(void) const
{
  if(!P)
    return PARSER_NOT_INITIALIZED;

  return P->Error;
}

/*******************************************************************************
   Function: PrintErrorCode()

Description: Prints out the error code, to the command prompt

     Inputs: N/A

    Outputs: N/A
*******************************************************************************/
void Parser::PrintErrorCode(void) const
{
  printf("%s: ", __FILE__);
  if(!P)
    printf("Parser was not initialized.\n");
  else
    switch(P->Error)
    {
      case 0:
        printf("No errors\n");
        break;
      case PARSER_COULD_NOT_OPEN_FILE:
        if(P->File)
          printf("Could not open |%s|\n", P->File);
        else
          printf("No file was specifed.\n");
        break;
      case PARSER_OUT_OF_MEMORY:
        printf("Memory Allocation failed.\n");
        break;
      case PARSER_END_OF_FILE:
        if(P->File)
          printf("Reached the end of %s\n", P->File);
        else
          printf("Reached the end of the memory\n");
        break;
      case PARSER_MEMORY_NOT_VALID:
        printf("Memory fed into LoadMemory wasn't valid\n");
        break;
      case PARSER_GRAB_TOKEN_IGNORE:
        printf("GrabToken() was called from a callback attached to a token ");
        printf("with Ignore set to true. This is not supported.\n");
        break;
    };
}

/*******************************************************************************
   Function: MemoryUsage()

Description: Returns the number of bytes the parser is useing.

     Inputs: N/A

    Outputs: Number of bytes the parser is using.
*******************************************************************************/
int Parser::MemoryUsage(void) const
{
  int size = 0;
  int tset;
  int sep;
  int start, end;

  if(!P)
    return 0;

  size += sizeof(Parser_str);
  size += sizeof(TokenSet_str) * P->NumOfTokenSet;
  if(P->SP)
  {
    if(!(P->Flags & PARSING_MEM))
      size += P->BufSize + P->LongestSep + 1;
    else
      if(!(P->Flags & DONT_FREE_SP))
        size += P->ActualSize + 1;
  }

  if(!(P->Flags & DONT_FREE_FILE) && P->File)
    size += strlen(P->File) + 1;

  for(tset = 0; tset < P->NumOfTokenSet; ++tset)
  {
    size += sizeof(Separator_str) * P->TSets[tset].NumOfSep;

      for(sep = 0; sep < P->TSets[tset].NumOfSep; ++sep)
        if(P->TSets[tset].Sep[sep].Flags & FREE_TOKEN)
        {
          if(P->Flags & PARSER_WILDCARD &&
             P->TSets[tset].Sep[sep].Flags & WILD_FLAGS)
          {
            end = 0;
            while(P->TSets[tset].Sep[sep].Token[end]) ++end;
            start = ++end;
            while(P->TSets[tset].Sep[sep].Token[end]) ++end;

            size += (end - start + 1) * 2;
          }
          else
          if(P->Flags & PARSER_REGULAR_EXPRESSIONS)
          {
            end = *(int *)P->TSets[tset].Sep[sep].Token;
            end += *(int *)(P->TSets[tset].Sep[sep].Token + end) + 1;
            size += end;
          }
          else
            size += P->TSets[tset].Sep[sep].Len + 1;
        }

    if(P->Flags & PARSER_HASH)
      size += sizeof(*P->TSets[0].Hash) * 256;

    if(P->Flags & PARSER_REGULAR_EXPRESSIONS)
    {
      size += sizeof(*P->RegEx);
      size += sizeof(*P->RegEx->Threads) * P->RegEx->NThreads;
      size += sizeof(*P->RegEx->Procs)   * P->RegEx->NProcs;
    }
  }

  return size;
}

/*******************************************************************************
   Function: GenericDiscard()

Description: Generic Parser callback designed to discard the next token. This
               is useful for removeing comments and such.

     Inputs: *buffer - Token that was recieved, and will be freed

    Outputs: 0 - Continue the search
*******************************************************************************/
char *GenericDiscard(Parser &p, char *buffer, void *)
{
  if(buffer)
    free(buffer);
  buffer = p.GrabToken();
  if(buffer)
    free(buffer);
  return 0;
}

/******************************************************************************
 *                     -- End of Public Parser library --                     *
 *                                                                            *
 * The following functions are just generic string minipulations functions    *
 * that are not aviable in the standard library, or opperate on different     *
 * principles.                                                                *
 ******************************************************************************/


/*******************************************************************************
   Function: RemoveWhiteSpaces()

Description: Removes all white spaces from a character array.
             White Spaces: Space
                           Tab
                           Return

             Procedure:
                Walk the array until we find the null terminator. If the current
                position contains a non null character, move it to the next
                valid position in the array. Once we find a white space
                character, we just move to the next character in the array.

     Inputs: *sp - Pointer to the string array to modify

    Outputs: -1 - Specifed string isn't valid
             New length of the array/position of null terminator
*******************************************************************************/
int RemoveWhiteSpaces(char *sp)
{
  register int i;   /* Current position we are checking for white space       */
  register int pos; /* Where we will store the next non white space character */

  if(!sp)
    return -1;

  for(i = 0, pos = 0; sp[i]; ++i)
    switch(sp[i])
    {
      case ' ' :    /* Space                                                  */
      case '\n':    /* new line                                               */
      case '\r':    /* carriage return                                        */
      case '\t':    /* Tab                                                    */
        break;
      default:
        sp[pos++] = sp[i];
        break;
    };

  sp[pos] = 0;      /* Mark the new end position of the character array       */

  return pos;
}

/*******************************************************************************
   Function: ToUpper()

Description: Makes every character in an array upper case

     Inputs: *sp - Array of characters to upper case

    Outputs: 0 - sp is null
             Pointer that was passed in.
*******************************************************************************/
char *ToUpper(char *sp)
{
  register int i;

  if(!sp)
    return 0;

  for(i = 0; sp[i]; ++i)
    if(sp[i] >= 'a' && sp[i] <= 'z')
      sp[i] = (char)(sp[i] + ('A' - 'a')); /* Prevents Borland Warning        */

  return sp;
}

/*******************************************************************************
   Function: ToLower()

Description: Makes every character in an array lower case

     Inputs: sp - Array of characters to lower case

    Outputs: 0 - sp is null
             Pointer that was passed in.
*******************************************************************************/
char *ToLower(char *sp)
{
  register int i;

  if(!sp)
    return 0;

  for(i = 0; sp[i]; ++i)
    if(sp[i] >= 'A' && sp[i] <= 'Z')
      sp[i] = (char)(sp[i] + ('a' - 'A')); /* Prevents Borland Warning        */

  return sp;
}

/*******************************************************************************
   Function: Dup()

Description: Takes in a string, and creates a duplicate of it

     Inputs: *sp - pointer to the string to duplicate

    Outputs: 0 - sp is null, sp is empty, or the memory could not be allocated
             pointer to a copy of the string
*******************************************************************************/
char *Dup(const char *sp)
{
  if(!sp)
    return 0;
  return DupLen(sp, (int)strlen(sp));
}

/*******************************************************************************
   Function: DupLen()

Description: Takes in a string, and creates a duplicate of it

     Inputs: *sp  - pointer to the string to duplicate
              len - Length of the string. The null terminator is automatically
                      attached. IE, you can juse use strlen(sp)

    Outputs: 0 - sp is null, sp is empty, or the memory could not be allocated
             pointer to a copy of the string
*******************************************************************************/
char *DupLen(const char *sp, int len)
{
  char *buffer;

  if(!sp || !len)
    return 0;

  if((buffer = (char *)malloc((unsigned int)++len)) == 0)
    return 0;

  memcpy(buffer, sp, (unsigned int)len);

  return buffer;
}

/*******************************************************************************
   Function: DupRange()

Description: Duplicates a specifec part of a string

     Inputs: *sp   - String to copy the data from
             start - Where in the start to copy data from
             end   - Where to stop copying data

    Outputs: 0 - sp is null, start == end, or the memory could not be allocated
             pointer to a copy of the string
*******************************************************************************/
char *DupRange(const char *sp, int start, int end)
{
  int len = end - start;
  char *buffer;

  if(!sp || !len)
    return 0;

  if((buffer = (char *)malloc((unsigned int)len + 1)) == 0)
    return 0;

  memcpy(buffer, sp + start, (unsigned int)len);
  buffer[len] = 0;

  return buffer;
}

/*******************************************************************************
   Function: DupRangeFile()

Description: Duplicates a specific part of a file

     Inputs: *file  - Name of the file to copy data from
              start - Where in the start to copy data from
              end   - Where to stop copying data

    Outputs: 0 - Error occured
             Pointer to the copy of the specifed part of the string
*******************************************************************************/
char *DupRangeFile(const char *file, int start, int end)
{
  int   len = end - start;
  char *buffer;
  FILE *fp    ;

  if(!file)
    return 0;

  if((fp = fopen(file, "rb")) == 0)
    return 0;

  if((buffer = (char *)malloc((unsigned int)len + 1)) == 0)
  {
    fclose(fp);
    return 0;
  }

  fseek(fp, start, 0);
  fread(buffer, 1, (unsigned int)len, fp);
  fclose(fp);

  buffer[len] = 0;

  return buffer;
}

/*******************************************************************************
   Function: Cmp()

Description: Case insensitive Compairision between two strings. All white
               spaces are removed, and then all characters are converted to
               lower case before the strings are compaired.

     Inputs: *osp  - First String to compare
             *osp2 - Second string to compaire

    Outputs: 1 - Strings are the same
             0 - Strings are not the same
*******************************************************************************/
char Cmp(const char *osp, const char *osp2)
{
  int i = 0;
  char *sp, *sp2;

  if(!osp || !osp2)
    return 0;

  sp  = Dup(osp);
  sp2 = Dup(osp2);

  if(RemoveWhiteSpaces(sp) != RemoveWhiteSpaces(sp2))
  {
    free(sp);
    free(sp2);
    return 0;
  }

  ToLower(sp);
  ToLower(sp2);

  while(sp[i] && sp[i] == sp2[i]) ++i;

  if(sp[i])
    i = 0;
  else
    i = 1;

  free(sp);
  free(sp2);

  return (char)i;
}



/******************************************************************************
 *                     -- End of Public String library --                     *
 *                                                                            *
 * The following functions are private parser functions. They should never be *
 *   called externally, since they make several assumptions about the sate of *
 *   the parser. Doing so will have undefined results.                        *
 ******************************************************************************/

/*******************************************************************************
   Function: GrabNextChunk()

Description: Grabs the next chunk of data from the file. If some data was
                left in P->SP, it will be preserved.

     Inputs: N/A

    Outputs: N/A - Updates P->SP to contain the next chunk of data.
*******************************************************************************/
void GrabNextChunk(Parser_str *P)
{
  int len;
  unsigned char *sp;

  if(P->Flags & END_OF_FILE){P->Error = PARSER_END_OF_FILE;         return;}
  if(!P->File)              {P->Error = PARSER_COULD_NOT_OPEN_FILE; return;}

    /* If P->SP is not valid, or we have to resize the buffer, do so.         */
  if(!P->SP || (P->Flags & ALLOC_FLAG))
  {
    if(!P->SP)
      P->SP = (BYTE *)malloc((unsigned int)(P->BufSize + P->LongestSep + 1));
    else
      P->SP = (BYTE *)realloc(P->SP,(unsigned int)(P->BufSize+P->LongestSep+1));

    P->Flags &= ~ALLOC_FLAG;

    if(!P->SP){P->Error = PARSER_OUT_OF_MEMORY; return;}
  }

    /* Make a copy of sp. This is easier then dealing with a index here.      */
  sp  = P->SP;
    /* Calculate how many bytes in the buffer have not been processed yet.    */
  len = P->ActualSize - P->SPos;

    /* If len is greater then the padding size, we cannot copy all of the     *
     *   leftover bytes to the start of the buffer. As such, we need to record*
     *   the location to P->CopyFPos, if we haven't done so already. This tell*
     *   use where the start of the next token is, so we don't need to a      *
     *   excessive amount of file in memory.                                  */
  if(len >= P->LongestSep && P->LongestSep)
  {
    if(P->CopyFPos == -1)
      P->CopyFPos = P->FPos - P->ActualSize + P->SPos;

      /* Update len so it no longer exceeds the padding area                  */
    len = P->LongestSep - 1;
  }

  if(len)
  {
      /* Copy any leftover bits to the start of the buffer                    */
    if(P->ActualSize - len)
    {
      if(len < P->ActualSize - len)
        memcpy(sp, sp + P->ActualSize - len, (unsigned int)len);
      else
        memmove(sp, sp + P->ActualSize - len, (unsigned int)len);
    }

      /* Shift sp to point at the byte directly after the copied data.        */
    sp += len;
  }

    /* If the file was not opend yet, open it.                                */
  if(!P->FP)
  {
    P->FP = fopen(P->File, "rb");
    if(!P->FP){P->Error = PARSER_COULD_NOT_OPEN_FILE;return;}
    if(P->FPos)
      fseek(P->FP, P->FPos, 0);
    P->Flags &= ~FPOS_MODIFIED;
  }

    /* If P->FPos was modified (and thus P->FPos != ftell(P->FP), update our  *
     *   position in the file to match P->FPos.                               */
  if(P->Flags & FPOS_MODIFIED)
  {
    fseek(P->FP, P->FPos, 0);
    P->Flags &= ~FPOS_MODIFIED;
  }

    /* Read the specified number of bytes into the buffer. P->CurBufSize      *
     *   contains the number of bytes that were read in from the file. This   *
     *   is equal to how many bytes we can processes from the file each time. */
  P->CurBufSize = (int)fread(sp, 1, (unsigned int)P->BufSize, P->FP);

    /* If we are suposed to close the file, do so.                            */
  if(P->Flags & PARSER_CLOSE_FILE)
  {
    fclose(P->FP);
    P->FP = 0;
  }

    /* Write a null terminator to the end of the string for convience reasons *
     *   This should cause each token sep to break, if they are compaired to  *
     *   area after the buffer. This will happen when we hit the end of the   *
     *   file and we are comparing the token seps against the end of the      *
     *   buffer, since there is no longer any padding.                        */
  sp[P->CurBufSize] = 0;

    /* Update where we are in the file.                                       */
  P->FPos += P->CurBufSize;

    /* SP Position is now at the start of the buffer.                         */
  P->SPos = 0;

    /* Calculate how much memor is actually in the buffer                     */
  P->ActualSize = P->CurBufSize + len;

  if(!P->CurBufSize || P->CurBufSize < P->BufSize)
  {
      /* If we didn't read anything from the file, or we were short changed,  *
       *   then we are at the end of the file. Mark the flag accordingly.     *
       *   This will cause the next GrabNextChunk() call to set the error code*
       *   to PARSER_END_OF_FILE, which siginifies we don't have anything     *
       *   else to process.                                                   */
    P->Flags |= END_OF_FILE;

      /* No more data in the file, so go ahead and give the other functions   *
       *   permission to read into the padding area.                          */
    P->CurBufSize = P->ActualSize;

      /* Go ahead and close the file if it's open, since we no longer need it.*/
    if(P->FP)
    {
      fclose(P->FP);
      P->FP = 0;
    }
  }
  else
  if(P->ActualSize < P->CurBufSize + P->LongestSep - 1)
  {
      /* We read the requested number of bytes from the file; however, we     *
       *   have not filled the padding area of the buffer. As such, we cannot *
       *   safely compare the token seps against the buffer. As such, update  *
       *   P->CurBufSize to reflect how far into the buffer the other         *
       *   functions can advance.                                             */

      /* P->LongestSep - 1 is the padding area size. We subtract len from this*
       *   since we originally added len to P->CurBufSize (since we assumed   *
       *   we read enough data from the file).                                */
    P->CurBufSize -= P->LongestSep - 1 - len;

      /* Note: If P->CurBufSize is < 0, this function will be called multiple *
       *         times until the padding is filled, or we hit the end of the  *
       *         file. We do not recursivly call GrabNextChunk() here, since  *
       *         this will increase the chance of a stack overflow. This      *
       *         situation is very uncommon, so the performance differnce is  *
       *         negligible. This will only happen if P->BufSize is less then *
       *         the padding size.                                            */
  }
}

/*******************************************************************************
   Function: RebuildHash()

Description: Builds or Rebuilds a hash table for all the token separators.

     Inputs: tset - Token set to generate the hash table for.

    Outputs: N/A
*******************************************************************************/
void RebuildHash(Parser_str *P, int tset)
{
  unsigned int  i;
  unsigned char c;
  TokenSet_str *set = P->TSets + tset;

  if(!set->Hash)
  {
    set->Hash = (unsigned int *)malloc(sizeof(*P->TSets[0].Hash) * 256);
    if(!set->Hash){P->Error = PARSER_OUT_OF_MEMORY; return;}
  }

  memset(set->Hash, -1, sizeof(*P->TSets[0].Hash) * 256);

  SortTokenSet(P, tset);

  for(i = 0; i < (unsigned int)set->NumOfSep; ++i)
  {
    c = set->Sep[i].Token[0]; FLAG_TOLOWER(c);
    if(set->Hash[c] == ~(unsigned int)0)
      set->Hash[c] = i;
  }
}

/*******************************************************************************
   Function: EnableWildcard()

Description: Enables Wildcard support for the specified token separator. First,
               the wildcard mask is generated. 2nd, the beginning and ending
               wildcard symbols are removed, and then the original string is
               attached to the end of the modified string.

     Inputs: sep - Sep to format for wildcard support

    Outputs: N/A
*******************************************************************************/
void EnableWildcard(Separator_str *sep)
{
  unsigned char *token;
  int read;
  int write;

    /* Clear all the WILD_* flags.                                            */
  sep->Flags &= ~WILD_FLAGS;

    /* Locate all wildcards in the string, and mark where then they are.      */
  for(read = 0; read < sep->Len; ++read)
    if(sep->Token[read] == '*')
    { /* Extra braces to remove compiler warnings */
      if(!read)
        sep->Flags |= WILD_START;
      else
      if(read == sep->Len - 1)
        sep->Flags |= WILD_END;
      else
      if(sep->Token[read - 1] != '*')
        sep->Flags |= WILD_MIDDLE;
   }

    /* If no wildcards were found, we don't have to do any special formatting.*/
  if(!(sep->Flags & WILD_FLAGS))
    return;

    /* Allocate room for the new string. The first half of the new string will*
     *   have the formated version of the string, while the second half will  *
     *   contain the original string.                                         */
  token = (unsigned char *)malloc((unsigned int)sep->Len * 2 + 2);

    /* Collapse any strands of '*' into just one '*'. IE, "a***b" -> "a*b".   *
     *   This also removes any leading '*'                                    */
  for(read = 0, write = 0; sep->Token[read]; ++read)
    if(sep->Token[read] != '*')
      token[write++] = sep->Token[read];
    else
      if(write != 0 && token[write - 1] != '*')
        token[write++] = '*';

    /* Remove any wild cards at the end of the string                         */
  if(token[write - 1] == '*')
    --write;
    /* Write the null terminator to the end of the formatted string.          */
  token[write++] = 0;

    /* Record the length of the formatted string                              */
  sep->Len = write - 1;

    /* Write the original string to the end of the formatted string.          */
  for(read = 0; sep->Token[read]; ++read)
    token[write++] = sep->Token[read];
  token[write] = 0;

  if(sep->Flags & FREE_TOKEN)
    free(sep->Token);
  sep->Flags |= FREE_TOKEN;

  sep->Token = token;
}

/*******************************************************************************
   Function: DisableWildcard()

Description: Restores the original sep->Token

     Inputs: sep - Separator to remove wildcard support from.

    Outputs: N/A
*******************************************************************************/
void DisableWildcard(Separator_str *sep)
{
  int len;
  int read;
  int write;
  unsigned char *token;

    /* If the string does not contain any wildcards, or it was not formatted, *
     *   we don't have to do anything.                                        */
  if(!(sep->Flags & WILD_FLAGS))
    return;

    /* Clear the wildcard flags                                               */
  sep->Flags &= ~WILD_FLAGS;

    /* Locate the starting and ending location of the original string.        *
     *   write = ending location                                              *
     *   read  = starting location (where to start reading)                   */
  write = 0;
  while(sep->Token[write]) ++write;
  read = ++write;
  while(sep->Token[write]) ++write;

  len = write - read;

  token = (unsigned char *)malloc((unsigned int)len + 1);

  for(write = 0; sep->Token[read]; ++read, ++write)
    token[write] = sep->Token[read];
  token[write] = 0;
  sep->Len = len;

  free(sep->Token);
  sep->Token = token;
}

/*******************************************************************************
   Function: SortTokenSet()

Description: Sorts the token separators in the specified token set. Insertion
               sort is used due to it's ease of use and efficency on small
               sets of data.

             Sorted based on the first character, followed by the length of the
               string. Namely
               Unsorted:
                 "foo"
                 "foobar"
                 "bar"

               Sorted:
                 "bar"    - Sort by first letter
                 "foobar" - Longer seps can be a superset of smaller ones
                 "foo"

             Sorting Requirements:
               Stable     - Preserve precedence of wildcard token seps
               In-Place   - Reduce parser memory utilization

     Inputs: tset - Token Set to sort

    Outputs: N/A
*******************************************************************************/
void SortTokenSet(Parser_str *P, int tset)
{
  unsigned int a, b;
  int i, j;
  TokenSet_str *set = P->TSets + tset;

  for(i = 1; i < set->NumOfSep; ++i)
    for(j = i; j; --j)
    {
      a = set->Sep[j - 1].Token[0]; FLAG_TOLOWER(a);
      b = set->Sep[j - 0].Token[0]; FLAG_TOLOWER(b);
      if(a > b || (a == b && set->Sep[j - 1].Len < set->Sep[j - 0].Len))
      {
        Separator_str swap = set->Sep[j - 1];
        set->Sep[j - 1]    = set->Sep[j];
        set->Sep[j - 0]    = swap;
      }
      else
        break;
    }
}

/*******************************************************************************
   Function: GTChar()

Description: GrabToken() helper function. Simplest version of the search
               algorithm without any extra options enabled.

     Inputs: N/A

    Outputs: 0 - no valid Token Separators are present
             Pointer to a new character array containing the token
*******************************************************************************/
char *GTChar(Parser_str *P)
{
  unsigned char *buffer; /* Char * that stores the string we will return      */
  unsigned char *token ; /* Token we are currently working with               */
  unsigned char  curc  ; /* Current character we are working with             */
  int            spos  ; /* Tracks the Character in SP we are checking        */
  int            sepnum; /* Tracks the current Separator Token to check       */

  for(; !P->Error; GrabNextChunk(P))
    for(spos = P->SPos; spos < P->CurBufSize; ++spos)
      for(curc = P->SP[spos], sepnum = 0; sepnum < P->NumOfSep; ++sepnum)
        if(curc == *P->CurSep[sepnum].Token)
        {
          token  = P->CurSep[sepnum].Token + 1;
          buffer = P->SP + spos + 1;
          while(*token && *buffer == *token){++buffer;++token;}

          if(*token)
            continue;

          buffer = ProcessToken(P, P->CurSep + sepnum, &spos);
          if(buffer)
            return (char *)buffer;

          if(spos > P->CurBufSize || !P->CurBufSize)
            break;

          sepnum = -1;
          curc   = P->SP[spos];
        }

  return GrabLeftover(P);
}

/*******************************************************************************
   Function: GTCharCaseIns()

Description: GrabToken() helper function. Case insensitive version of GTChar().
               The main difference is in the use of the TOLOWER() macro.
               TOLOWER() was used over TOUPPER(), since lowercase characters
               are typically more common then upper case characters. IE, less
               math has to be done.

     Inputs: N/A

    Outputs: 0 - no valid Token Separators are present
             Pointer to a new character array containing the token
*******************************************************************************/
char *GTCharCaseIns(Parser_str *P)
{
  unsigned char *buffer; /* Char * that points to where we are in P->SP       */
  unsigned char *token ; /* Char * that points to the sep we are comparing    */
  unsigned char  curc  ; /* Stores P->SP[spos]. Reduces memory lookups.       */
  unsigned char  t     ; /* Stores the case insensitive char from the sep     */
  unsigned char  b     ; /* Stores the case insensitive char from P->SP       */
  int            spos  ; /* Tracks the Character in SP we are checking        */
  int            sepnum; /* Tracks the current Separator Token to check       */

  for(; !P->Error; GrabNextChunk(P))
    for(spos = P->SPos; spos < P->CurBufSize; ++spos)
    {
      curc = P->SP[spos];
      curc = TOLOWER(curc);

      for(sepnum = 0; sepnum < P->NumOfSep; ++sepnum)
      {
        t = *P->CurSep[sepnum].Token;
        t = TOLOWER(t);
        if(curc == t)
        {
          token  = P->CurSep[sepnum].Token + 1;
          buffer = P->SP + spos + 1;
          t = *token;  t = TOLOWER(t);
          b = *buffer; b = TOLOWER(b);
          while(t && b == t)
          {
            ++buffer; b = *buffer; b = TOLOWER(b);
            ++token;  t = *token;  t = TOLOWER(t);
          }

          if(t)
            continue;

          buffer = ProcessToken(P, P->CurSep + sepnum, &spos);
          if(buffer)
            return (char *)buffer;

          if(spos > P->CurBufSize || !P->CurBufSize)
            break;

          sepnum = -1;
          curc   = P->SP[spos];
          curc   = TOLOWER(curc);
        }
      }
    }

  return GrabLeftover(P);
}

/*******************************************************************************
   Function: GTCharHash()

Description: GrabToken() helper function. Case sensitive search with Hash
               optimization. This is the fastest search algorithm in the
               parser; however, it also consumes a lot of additional memory.

     Inputs: N/A

    Outputs: 0 - no valid Token Separators are present
             Pointer to a new character array containing the token
*******************************************************************************/
char *GTCharHash(Parser_str *P)
{
  unsigned char *buffer; /* Points to our place in P->SP, during comparison   */
  unsigned char *token ; /* Points to the token sep we are comparing.         */
  unsigned char  curc  ; /* Current character we are working with             */
  int            spos  ; /* Tracks the Character in SP we are checking        */
  unsigned int   sepnum; /* Tracks the current Separator Token to check       */

  for(; !P->Error; GrabNextChunk(P))
    for(spos = P->SPos; spos < P->CurBufSize; ++spos)
    {
      curc   = P->SP[spos];
      sepnum = P->TSets[P->CurTokenSet].Hash[curc];

      if(sepnum == ~(unsigned int)0)
        continue;

      for(;(int)sepnum < P->NumOfSep&&P->CurSep[sepnum].Token[0]==curc;++sepnum)
      {
        token  = P->CurSep[sepnum].Token + 1;
        buffer = P->SP + spos + 1;
        while(*token && *buffer == *token){++buffer;++token;}

        if(*token)
          continue;

        buffer = ProcessToken(P, P->CurSep + sepnum, &spos);
        if(buffer)
          return (char *)buffer;

        if(spos > P->CurBufSize || !P->CurBufSize)
          break;

        curc   = P->SP[spos];
        sepnum = P->TSets[P->CurTokenSet].Hash[curc];
        if(sepnum == ~(unsigned int)0)
          break;
        --sepnum;
      }
    }

  return GrabLeftover(P);
}

/*******************************************************************************
   Function: GTCharHashCaseIns()

Description: GrabToken() helper function. Case insensitive version of GTCharHash

     Inputs: N/A

    Outputs: 0 - no valid Token Separators are present
             Pointer to a new character array containing the token
*******************************************************************************/
char *GTCharHashCaseIns(Parser_str *P)
{
  unsigned char *buffer; /* Char * that points to where we are in P->SP       */
  unsigned char *token ; /* Char * that points to the sep we are comparing    */
  unsigned char  curc  ; /* Stores P->SP[spos]. Reduces memory lookups.       */
  unsigned char  t     ; /* Stores the case insensitive char from the sep     */
  unsigned char  b     ; /* Stores the case insensitive char from P->SP       */
  int            spos  ; /* Tracks the Character in SP we are checking        */
  unsigned int   sepnum; /* Tracks the current Separator Token to check       */

  for(; !P->Error; GrabNextChunk(P))
    for(spos = P->SPos; spos < P->CurBufSize; ++spos)
    {
      curc   = P->SP[spos]; curc = TOLOWER(curc);
      sepnum = P->TSets[P->CurTokenSet].Hash[curc];

      if(sepnum == ~(unsigned int)0)
        continue;

      for(; (int)sepnum < P->NumOfSep; ++sepnum)
      {
        if(TOLOWER(P->CurSep[sepnum].Token[0]) != curc)
          break;

        token  = P->CurSep[sepnum].Token + 1;
        buffer = P->SP + spos + 1;
        t = *token;  t = TOLOWER(t);
        b = *buffer; b = TOLOWER(b);
        while(t && b == t)
        {
          ++buffer; b = *buffer; b = TOLOWER(b);
          ++token;  t = *token;  t = TOLOWER(t);
        }

        if(t)
          continue;

        buffer = ProcessToken(P, P->CurSep + sepnum, &spos);
        if(buffer)
          return (char *)buffer;

        if(spos > P->CurBufSize || !P->CurBufSize)
          break;

        curc   = P->SP[spos]; curc = TOLOWER(curc);
        sepnum = P->TSets[P->CurTokenSet].Hash[curc];
        if(sepnum == ~(unsigned int)0)
          break;
        --sepnum;
      }
    }

  return GrabLeftover(P);
}

/*******************************************************************************
   Function: GTCharWild()

Description: GrabToken() helper function. This function handles wild cards,
               as well as the other options. Wildcard based searching is slow
               enough due to the required ForwardSearch(). As such, breaking
               this function up into several other varients will not have a
               noticeable effect on performance.

     Inputs: findstart - For recursive calls. Should the start of the next
                           token be returned, or should we return the actual
                           token?

    Outputs: 0 - no valid Token Separators are present
             Pointer to a new character array containing the token
*******************************************************************************/
char *GTCharWild(Parser_str *P, int findstart)
{
  unsigned char *buffer; /* Char * that points to where we are in P->SP       */
  unsigned char *token ; /* Char * that points to the sep we are comparing    */
  unsigned char  curc  ; /* Stores P->SP[spos]. Reduces memory lookups.       */
  unsigned char  t     ; /* Stores the case insensitive char from the sep     */
  unsigned char  b     ; /* Stores the case insensitive char from P->SP       */
  int            spos  ; /* Tracks the Character in SP we are checking        */
  unsigned int   sepnum; /* Tracks the current Separator Token to check       */
  long           end   ; /* Ending or Starting pos of the token               */

  for(; !P->Error; GrabNextChunk(P))
    for(spos = P->SPos; spos < P->CurBufSize; ++spos)
    {
      curc = P->SP[spos];
      FLAG_TOLOWER(curc);

      if(P->Flags & PARSER_HASH)
      {
        sepnum = P->TSets[P->CurTokenSet].Hash[curc];
        if(sepnum == ~(unsigned int)0)
          continue;
      }
      else
        sepnum = 0;
      for(; sepnum < (unsigned int)P->NumOfSep; ++sepnum)
      {
        token = P->CurSep[sepnum].Token;
        t     = *token;

        FLAG_TOLOWER(t);

        if(curc != t)
        {
          if(P->Flags & PARSER_HASH)
            break;
          continue;
        }

        t = *++token;        FLAG_TOLOWER(t);
        b = P->SP[spos + 1]; FLAG_TOLOWER(b);

        if(P->CurSep[sepnum].Flags & WILD_FLAGS)
        {
          end =ForwardSearch(P, spos,
                             findstart&&!(P->CurSep[sepnum].Flags&IGNORE_TOKEN),
                             P->CurSep + sepnum);
          if(end == -1)
            continue;
        }
        else
        {
          end = spos;
          while(t && b == t)
          {
            ++token; t = *token;         FLAG_TOLOWER(t);
            ++end;   b = P->SP[end + 1]; FLAG_TOLOWER(b);
          }

          if(t)
            continue;
        }

        if(findstart && !(P->CurSep[sepnum].Flags & IGNORE_TOKEN))
          return (char *)spos;

        buffer = ProcessTokenWild(P, P->CurSep + sepnum, &spos, &end);
        if(buffer)
          return (char *)buffer;

        if(spos >= P->CurBufSize || !P->CurBufSize)
          break;

        if(spos == -1)
        {
          spos   = P->SPos - 1;
          sepnum = (unsigned int)(P->NumOfSep - 1);

          if(spos == -1)
            break;
        }
        else
          sepnum = ~(unsigned int)0;

        curc = P->SP[spos];
        FLAG_TOLOWER(curc);
        if(P->Flags & PARSER_HASH)
        {
          sepnum = P->TSets[P->CurTokenSet].Hash[curc];
          if(sepnum == ~(unsigned int)0)
            break;
          --sepnum;
        }
      }
    }

  if(findstart)
    return (char *)P->ActualSize;

  return GrabLeftover(P);
}

/*******************************************************************************
   Function: ForwardSearch()

Description: Starts from spos, and searches through the rest of the file
               for the specified token separator. If !findstart, the end
               of the token will be returned. Otherwise, the start of the
               token will be found. Wildcards and case insensitivity are
               handled.

     Inputs:  spos      - Where to start searching in P->SP
              findstart - Should the start or end of the token be returned?
             *sep       - Token Separator to look for

    Outputs: 0 - Not Found
             if(!findstart)
               End of the specified token (in file space)
             else
               Start of the specified token (in file space)
*******************************************************************************/
long ForwardSearch(Parser_str *P, int spos, int findstart, Separator_str *sep)
{
  unsigned char *token = sep->Token;
  void *state;
  long  start = P->FPos - P->ActualSize + spos; /* Locate the start of token  */
  int   stuffbefore = 1;
  int   i, t, b;
  long  ret;

  if(P->SPos == spos && P->CopyFPos == -1)
    stuffbefore = 0;

  for(t = *token, b = P->SP[spos]; t; t = *++token, b = P->SP[++spos])
  {
    if(t == '*')
    {
      t = *++token;
      FLAG_TOLOWER(t);
      break;
    }
    FLAG_TOLOWER(t);
    FLAG_TOLOWER(b);

    if(t != b)
      return -1;
  }
  state = PreserveBufferState(P);
    /* Prevent GrabNextToken() from reloading something we scanned            */
  P->SPos = spos;

  if(t)
    for(; !P->Error; GrabNextChunk(P), spos = 0)
    {
      for(; t && spos < P->CurBufSize; ++spos)
      {
        b = P->SP[spos];
        FLAG_TOLOWER(b);

        if(b != t)
          continue;

        for(i = 1; token[i] && token[i] != '*'; ++i)
        {
          int t2 = token[i];
          int b2 = P->SP[spos + i];
          FLAG_TOLOWER(t2);
          FLAG_TOLOWER(b2);
          if(t2 != b2)
            break;
        }

        if(token[i] && token[i] != '*')
          continue;

        token  += i;
        t       = *token;
        spos   += i - 1;
        P->SPos = spos + 1;

        if(t == '*') t = *++token;
        FLAG_TOLOWER(t);
      }

      if(!t)
        break;
    }

  if(t)
    ret = -1;
  else
  if(findstart || (!(sep->Flags & WILD_START) && stuffbefore))
    ret = start;
  else
  if((sep->Flags & WILD_START || !stuffbefore) && !(sep->Flags & WILD_END))
    ret = P->FPos - P->ActualSize + spos;
  else
  {
    void *tset = 0;

    if(!(sep->Flags & IGNORE_TOKEN) && sep->SwitchTo != -1)
    {
      tset = PreserveTSetHistory(P);
      SetTokenSet(P, sep->SwitchTo);
    }

    spos = (int)GTCharWild(P, 1);
    RestoreTSetHistory(P, tset);
    ret = P->FPos - P->ActualSize + spos;
  }

  RestoreBufferState(P, state);
  return ret;
}

/*******************************************************************************
   Function: ProcessToken()

Description: Handles a Token Sepeator when ever one is found.

     Inputs: *sep  - Pointer to the Separator_str that was found.
             *spos - Position in the string where the separator was found.
                       This function will most likely update the position in
                       the string.

    Outputs: 0  - GrabToken() should not return anything. Search for the next
                  token.
             1+ - Pointer to the token to return.
*******************************************************************************/
BYTE *ProcessToken(Parser_str *P, Separator_str *sep, int *spos)
{
  char *buffer = 0;

  if(sep->Flags & IGNORE_TOKEN)
  {
      /* We are suposed to ignore the token. Update our position in the stream*
       *   and call any callbacks. Nothing else should happen.                */
    *spos += sep->Len;

      /* Prevent a callback from calling GrabToken(), since it will start a   *
       *   infinate loop. Also, GetFilePostion() won't be right.              *
       *                                                                      *
       * Note: P->Error should be set to 0. If it isn't, we would not be able *
       *         to get here. As such, we do not need to preserve it's state. */
    P->Error = PARSER_GRAB_TOKEN_IGNORE;
    if(sep->Fun)
      (*sep->Fun)(*P->PClass, 0, sep->Params);
    P->Error = 0;
  }
  else
  if(*spos == P->SPos && P->CopyFPos == -1)
  {
      /* The token was found, and nothing was before it. Update our position, *
       *   call any callbacks, and then duplicate the string.                 */
    P->SPos += sep->Len;
    SetTokenSet(P, sep->SwitchTo);

    P->LParam = sep->Params;

    if(sep->Fun)
    {
      buffer = DupRange((char *)P->SP, P->SPos - sep->Len, P->SPos);
      buffer = (*sep->Fun)(*P->PClass, buffer, sep->Params);
      if(!(sep->Flags & RETURN_TOKEN))
        if(buffer)
        {
          free(buffer);
          buffer = 0;
        }

      *spos = P->SPos;
    }
    else
    if(sep->Flags & RETURN_TOKEN)
      buffer = DupRange((char *)P->SP, P->SPos - sep->Len, P->SPos);
    else
      *spos = P->SPos;
  }
  else
  {
      /* Stuff was found before the token. As such, return it.                */
    if(P->CopyFPos == -1)
      buffer = DupRange((char *)P->SP, P->SPos, *spos);
    else
    {
      buffer = PDupRangeFile(P, P->CopyFPos, P->FPos + *spos - P->ActualSize);
      P->CopyFPos = -1;
    }

    P->SPos = *spos;
  }

  return (BYTE *)buffer;
}

/*******************************************************************************
   Function: ProcessTokenWild()

Description: When ever a token is found in GTCharWild(), this function is
               called. This function handles all logic attached to sep, and
               returns the token if it is suposed to be returned.

     Inputs: *sep  - Token that was found. Used to access the logic of the token
             *spos - Starting location of the found token
             *end  - Ending location of the found token

    Outputs: 0  - Token !returned, ignored, or a error.
             1+ - Pointer to the string to return.
*******************************************************************************/
BYTE *ProcessTokenWild(Parser_str *P, Separator_str *sep, int *spos, long *end)
{
  char *buffer = 0;

    /* If there are no wild cards, or if there is not any wild card in front  *
     *   of the token and stuff was found before it, we can process the token *
     *   normally.                                                            */
  if(P->Flags & PARSER_REGULAR_EXPRESSIONS)
  {
    if(P->SPos != *spos || P->CopyFPos != -1)
      return ProcessToken(P, sep, spos);
  }
  else
    if(!(sep->Flags & WILD_FLAGS) ||
      (!(sep->Flags & WILD_START) && (P->SPos != *spos || P->CopyFPos != -1)))
      return ProcessToken(P, sep, spos);

  if(sep->Flags & IGNORE_TOKEN)
  {
    int start = P->SPos;
    long copyfpos = P->CopyFPos;
    if(copyfpos == -1)
      copyfpos = GetFilePosition(P);
    if(P->Flags & PARSING_MEM)
      *spos = *end;
    else
    {
      SetFilePosition(P, *end);
      if(P->SPos)
      {
        *spos   = P->SPos;
        P->SPos = start;
      }
      else
      {
        P->CopyFPos = copyfpos;
        *spos = -1;
      }
    }

    P->Error = PARSER_GRAB_TOKEN_IGNORE;
    if(sep->Fun)
      (*sep->Fun)(*P->PClass, 0, sep->Params);  /* Deref is required for DeSmet C support */
    P->Error = 0;
    return 0;
  }

  SetTokenSet(P, sep->SwitchTo);

  P->LParam = sep->Params;

  if(sep->Fun)
  {
    if(P->CopyFPos == -1)
    {
      if(*end - P->FPos <= 0)
        buffer = DupRange((char *)P->SP, P->SPos, *end - P->FPos+P->ActualSize);
      else
        buffer = PDupRangeFile(P, P->FPos - P->ActualSize + P->SPos, *end);
    }
    else
    {
      buffer = PDupRangeFile(P, P->CopyFPos, *end);
      P->CopyFPos = -1;
    }

    buffer = (*sep->Fun)(*P->PClass, buffer, sep->Params);
    *spos = P->SPos;
    if(!(sep->Flags & RETURN_TOKEN))
      if(buffer)
      {
        free(buffer);
        buffer = 0;
      }
  }
  else
  if(sep->Flags & RETURN_TOKEN)
  { /* Extra braces to remove compiler warning */
    if(P->CopyFPos == -1)
    {
      if(*end - P->FPos <= 0)
        buffer = DupRange((char *)P->SP, P->SPos, *end - P->FPos+P->ActualSize);
      else
        buffer = PDupRangeFile(P, P->FPos - P->ActualSize + P->SPos, *end);
    }
    else
    {
      buffer = PDupRangeFile(P, P->CopyFPos, *end);
      P->CopyFPos = -1;
    }
  }

  P->CopyFPos = -1;
  SetFilePosition(P, *end);
  *spos = -1;

  return (BYTE *)buffer;
}

/*******************************************************************************
   Function: GrabLeftover()

Description: Returns any left over data in the file

     Inputs: N/A

    Outputs: 0  - No data is left in SP or the file.
             1+ - Pointer to the token.
*******************************************************************************/
char *GrabLeftover(Parser_str *P)
{
  char *buffer;

  if(P->CopyFPos != -1)
    buffer = PDupRangeFile(P, P->CopyFPos, P->FPos);
  else
    buffer = DupRange((char *)P->SP, P->SPos, P->ActualSize);

  P->CopyFPos   = -1;
  P->CurBufSize =  0;
  P->ActualSize =  0;
  P->SPos       =  0;

  return buffer;
}

/*******************************************************************************
   Function: PDupRangeFile()

Description: Duplicates a specifec part of a file

     Inputs: start - Where in the start to copy data from
             end   - Where to stop copying data

    Outputs: Pointer to the copy of the specifed part of the string
*******************************************************************************/
char *PDupRangeFile(Parser_str *P, int start, int end)
{
  int   len = end - start;
  char *buffer;
  FILE *fp    ;

  if(!P->File || !len)
    return 0;

  if(P->FP)
    fp = P->FP;
  else
  if((fp = fopen(P->File, "rb")) == 0)
    return 0;

  if((buffer = (char *)malloc((unsigned int)len + 1)) == 0)
  {
    if(!P->FP)
      fclose(fp);
    return 0;
  }

  fseek(fp, start, 0);
  fread(buffer, 1, (unsigned int)len, fp);
  if(P->FP)
    fseek(P->FP, P->FPos, 0);
  else
    fclose(fp);

  buffer[len] = 0;

  return buffer;
}

/*******************************************************************************
   Function: ReadBinary()

Description: GrabBinary*() helper function. Read the specified number of bytes
               from P->SP. If the requested size is to large, the bytes will
               be read in from the file. Finally, the position in the buffer
               is updated.

     Inputs: *dest - Where to write the data to. Assumes size will fit into dest
              size - How many bytes to write to dest.

    Outputs: 0 - Error occured.
             1 - Wrote the requested number of bytes to dest.
*******************************************************************************/
int ReadBinary(Parser_str *P, void *dest, int size)
{
  if(!P || P->Error)
    return 0;

  if(P->SPos + size > P->ActualSize)
  {
    /* If there are not enough bytes in the buffer to read the requested size,*
     *   read in the data from the file, and then set the position of the     *
     *   parser in the file to the spot directly after teh memory we read.    *
     * This was done since there is no garantee that the requested number of  *
     *   bytes would fit into the parsers buffer.                             */
    int   bytesread;
    int   fpos = GetFilePosition(P);
    FILE *fp;

    if(!P->File)
      return 0;

    if((fp = fopen(P->File, "rb")) == 0)
      return 0;

    fseek(fp, fpos, 0);
    bytesread = (int)fread(dest, 1, (unsigned int)size, fp);
    fclose(fp);

    if(bytesread != size)
      return 0;

    SetFilePosition(P, fpos + size);
  }
  else
  {
    memcpy(dest, P->SP + P->SPos, (unsigned int)size);
    P->SPos += size;
  }

  return 1;
}

/*******************************************************************************
   Function: PreserveBufferState()

Description: Preserves the values in P that relate to the state of P->SP. In
               order to reduce memory usage and increase, P->SP is not
               duplicated. Instead, all position related values are stored.

     Inputs: N/A

    Outputs: 0  - Could not allocate enough memory
             1+ - Pointer to the parser buffer state
*******************************************************************************/
void *PreserveBufferState(Parser_str *P)
{
  long *state = (long *)malloc(sizeof(long) * 8);

  if(!state) {P->Error = PARSER_OUT_OF_MEMORY; return 0;}

  state[0] = P->SPos;
  state[1] = P->FPos;
  state[2] = P->CopyFPos;
  state[3] = P->CurBufSize;
  state[4] = P->ActualSize;
  state[5] = P->BufSize;
  state[6] = P->Error;
  state[7] = P->Flags;

  return state;
}

/*******************************************************************************
   Function: RestoreBufferState()

Description: Restores the state of the parser, relating to P->SP. Since this
               function is designed to be called within GrabToken() or similar,
               we have to restore the buffer state to the exact state it was,
               according to the specified state.

     Inputs: *_state - State to set the parser to. This should only be a pointer
                         returned from PreserveBufferState().

    Outputs: N/A
*******************************************************************************/
void RestoreBufferState(Parser_str *P, void *_state)
{
  long *state = (long *)_state;

  if(!P || !state) return;

  if(P->FPos != state[1] || P->ActualSize != state[4])
  {
      /* Only rebuild P->SP, if one of the below cases is meet:               *
       *   Case 1: P->FPos != state->FPos                                     *
       *           Simple case. The file position changed, so we must update  *
       *   Case 2: P->ActualSize != state->ActualSize                         *
       *           If we read in the entire file, then P->FPos will not change*
       *           instead, P->ActualSize will decrease to fit the leftover   *
       *           data. As such, we will need to restore P->SP.              */
    P->Error      = state[6];
    P->Flags      = (state[7] | FPOS_MODIFIED) & ~END_OF_FILE;
    P->FPos       = state[1] - state[4];
    P->SPos       = state[4];
    P->BufSize    = state[4];
    P->ActualSize = state[4];
    GrabNextChunk(P);
  }

  P->SPos       = state[0];
  P->CopyFPos   = state[2];
  P->CurBufSize = state[3];
  P->BufSize    = state[5];
  P->Error      = state[6];
  P->Flags      = state[7];

  free(state);
}

/*******************************************************************************
   Function: PreserveTSetHistory()

Description: Preserves the current token set history. P->TSetHistory is
               duplicated, since the entire tset history can change.

             Stored as such:
               | History... | Pos | CurTokenSet |

     Inputs: N/A

    Outputs: Pointer to the Token Set State
*******************************************************************************/
void *PreserveTSetHistory(Parser_str *P)
{
  int *history;

  if(!P)
    return 0;

  history = (int *)malloc(sizeof(int) * (TSET_HISTORY_DEPTH + 2));

  if(!history)
    return 0;

  memcpy(history, P->TSetHistory, sizeof(int) * TSET_HISTORY_DEPTH);
  history[TSET_HISTORY_DEPTH + 0] = P->TSetHistoryPos;
  history[TSET_HISTORY_DEPTH + 1] = P->CurTokenSet;

  return history;
}

/*******************************************************************************
   Function: RestoreTSetHistory()

Description: Restores the token set hisotry to the specified state.

     Inputs: *state - TSet History State to set the parser to.

    Outputs: N/A
*******************************************************************************/
void RestoreTSetHistory(Parser_str *P, void *state)
{
  int *history = (int *)state;

  if(!P || !history)
    return;

  if(history[TSET_HISTORY_DEPTH + 1] != P->CurTokenSet)
    SetTokenSet(P, history[TSET_HISTORY_DEPTH + 1]);
  memcpy(P->TSetHistory, history, sizeof(int) * TSET_HISTORY_DEPTH);
  P->TSetHistoryPos = history[TSET_HISTORY_DEPTH + 0];

  free(state);
}
















/******************************************************************************
 *               -- In Progress Regular Expression Extension --               *
 *               -- In Progress Regular Expression Extension --               *
 *               -- In Progress Regular Expression Extension --               *
 *               -- In Progress Regular Expression Extension --               *
 *               -- In Progress Regular Expression Extension --               *
 *               -- In Progress Regular Expression Extension --               *
 *               -- In Progress Regular Expression Extension --               *
 *               -- In Progress Regular Expression Extension --               *
 *               -- In Progress Regular Expression Extension --               *
 ******************************************************************************/

/*******************************************************************************
   Function: PrintCompiled()

Description: Prints out a compiled byte code.
             Note: This function is commented out to remove warnings.
                     Namely, the "Unused local funciton" warning.

     Inputs: *data - Pointer to the compiled byte code to print out.

    Outputs: N/A
*******************************************************************************/
/*
void PrintCompiled(BYTE *data)
{
  int  i, j, k;
  BYTE c;

  printf("\n");
  printf("Compiled:\n");
  printf("  0:    Len: %d\n", GET_INT(data, 0));
  printf("  4:Threads: %d\n", GET_INT(data, 4));
  i = 8;
  do
  {
    printf("%3d: ", i);
    c = data[i++];
    switch(c)
    {
      case LITERAL: printf("%c\n", data[i++]);  break;
      case RANGE:
        printf("Range: |");
        for(j = 0; j < 32; ++j)
        {
          c = data[i++];

          if(j > 3 && j < 16)
            for(k = 0; k < 8; ++k)
              if((c >> k) & 1)
                printf("%c", j * 8 + k);
        }
        printf("|\n");
        break;

      case WILD:             printf("Wild\n");  break;
      case WORD_BOUNDRY:     printf("Word Boundry\n"); break;
      case WORD_BOUNDRY_NOT: printf("Not a Word Boundry\n"); break;
      case WORD_START:       printf("Word Start\n"); break;
      case WORD_END:         printf("Word End\n"); break;
      case LINE_START:       printf("Start of a Line\n"); break;
      case LINE_END:         printf("End of a Line\n"); break;
      case THREAD:           printf("Thread: %d\n",GET_INT(data, i));i+=4;break;
      case THREAD_NON_GREEDY:printf("Thread Non Greedy: %d\n", GET_INT(data, i));i+=4;break;
      case JUMP:             printf("Jump: %d\n",  GET_INT(data, i));i+=4;break;
      case PASS:             printf("Pass\n");  break;
      default:               printf("Error\n"); break;
    }
  }
  while(c != PASS);
  printf("\n");
}
*/

/*******************************************************************************
   Function: EnableRegEx()

Description: Compiles a string to the bytecode representation of the RegEx.
               Format
                 Len of Byte Code (4 bytes)
                 Max Threads      (4 bytes)
                 Command          (1 byte )
                 Param(s)         (x bytes)
                 ...
                 PASS             (1 byte )
                 Len of Token     (4 bytes) <- Len of Byte Code Points here
                 Original Token   (x bytes)

     Inputs: sep - Separator to add wildcard support to.

    Outputs: N/A
*******************************************************************************/
int EnableRegEx(Parser_str *P, Separator_str *sep)
{
  unsigned int   error;
  unsigned char *sp = sep->Token;

  error = InvalidRegEx(P, (char *)sp);
  if(error)
  {
    if(sep->Flags & FREE_TOKEN)
      free(sp);
    return error;
  }

  sep->Token = Compile(P, (char *)sp);

  if(sep->Flags & FREE_TOKEN)
    free(sp);
  sep->Flags |= FREE_TOKEN;

  if(!sep->Token)
    return PARSER_REGEX_COMPILE_ERROR;
  return 0;
}

/*******************************************************************************
   Function: DisableRegEx()

Description: Restores the original sep->Token. See EnableRegEx() for the
               layout of the memory.

     Inputs: sep - Separator to remove RegEx support from.

    Outputs: N/A
*******************************************************************************/
void DisableRegEx(Separator_str *sep)
{
  char *sp;
  int pos = *(int *)sep->Token;
  int len = *(int *)(sep->Token + pos);

  pos += 4;

  sp = DupLen((char *)sep->Token + pos, len);
  free(sep->Token);
  sep->Token = (unsigned char *)sp;
}

/*******************************************************************************
   Function: ForwardSearchReg()

Description: Searches through the string with the specified byte code

     Inputs:  spos     - Starting location in P->SP
             *data     - Byte code to test P->SP against
              prevchar - Previous character form P->SP

    Outputs: 0  - Specified RegEx did not match
             1+ - Absolute (file position) location of the end of the RegEx
*******************************************************************************/
long ForwardSearchReg(Parser_str *P, int spos, BYTE *data, BYTE prevchar)
{
  Thread_str *threads = P->RegEx->Threads;
  int   greed    = INT_MAX;
  int   end      = 0;  /* Location in the file where PASS was found */
  int   nthreads = 0;
  int   t;
  int  *procs    = P->RegEx->Procs;
  int   loop     = 1;
  void *state    = PreserveBufferState(P);
  BYTE  buf[3];

  memset(procs, 0, sizeof(int) * GET_INT(data, 0));

  threads[nthreads  ].DPos  = 8;
  threads[nthreads++].Greed = 0;

  buf[1] = prevchar;
  buf[2] = 0;

  if(spos >= P->ActualSize )
  {
    P->SPos = spos;
    GrabNextChunk(P);
    spos = 0;
  }

  if(!P->Error)
  {
    buf[2] = P->SP[spos++];

    do
    {
      for(; spos < P->ActualSize; ++spos, ++loop)
      {
        buf[0] = buf[1];
        buf[1] = buf[2];
        buf[2] = P->SP[spos];

        if(UpdateThreads(P, buf, data, threads, &nthreads, procs, loop, &greed))
        {
          end = P->FPos - P->ActualSize + spos - 1;
          for(t = 0; t < nthreads; ++t)
            if(threads[t].Greed > greed)
              POP_THREAD
        }

        if(!nthreads)
          break;
      }

      P->SPos = spos;
      GrabNextChunk(P);
      spos = 0;
    }
    while(!P->Error && nthreads);

    buf[0] = buf[1];
    buf[1] = buf[2];
    buf[2] = 0;

    if(UpdateThreads(P, buf, data, threads, &nthreads, procs, ++loop, &greed))
      end = P->FPos - 1;

    buf[0] = buf[1];
    buf[1] = 0;

    if(UpdateThreads(P, buf, data, threads, &nthreads, procs, ++loop, &greed))
      end = P->FPos;
  }

  RestoreBufferState(P, state);
  return end;
}

/*******************************************************************************
   Function: GTCharRegEx()

Description: Returns the next token in the stream, based on RegEx logic.
               This function works with the flags
                 PARSER_CASE_INSENSITIVE
                 PARSER_CLOSE_FILE
               This function does not work with and will disable the following
                 defines when Enable(PARSER_REGEX) is called
                 PARSER_HASH
                 PARSER_WILDCARD
                 PARSER_SORT_TOKEN_SEPS

             Note: RegEx's are not token separator delimited. IE, "*a*" will
                     return everything.

     Inputs: N/A

    Outputs: 0  - Error Occured or the End of the File was found
             1+ - Pointer to the new token
*******************************************************************************/
char *GTCharRegEx(Parser_str *P)
{
  BYTE        *buffer; /* Char * that points to where we are in P->SP         */
  long         end   ; /* Absolute ending location of a matching string       */
  int          spos  ; /* Tracks the Character in SP we are checking          */
  unsigned int sepnum; /* Tracks the current Separator Token to check         */
  BYTE         pchar;  /* Previous char                                       */

  for(; !P->Error; GrabNextChunk(P))
    for(spos = P->SPos; spos < P->ActualSize; ++spos)
    {
      pchar = PeekByte(P, spos - P->SPos - 1);
      for(sepnum = 0; sepnum < (unsigned int)P->NumOfSep; ++sepnum)
      {
        end = ForwardSearchReg(P, spos, P->CurSep[sepnum].Token, pchar);
        if(!end)
          continue;

        buffer = ProcessTokenWild(P, P->CurSep + sepnum, &spos, &end);
        if(buffer)
          return (char *)buffer;

        if(spos > P->CurBufSize || !P->CurBufSize)
          break;

        if(spos == -1)
        {
          spos   = P->SPos - 1;
          sepnum = (unsigned int)(P->NumOfSep - 1);
        }
        else
          sepnum = ~(unsigned int)0;

        pchar = PeekByte(P, spos - P->SPos - 1);
      }
    }

  return GrabLeftover(P);
}

/*******************************************************************************
   Function: HandleEscapes()

Description: Removes and stores all escape sequences from a RegEx in a
               seperate array.

     Inputs: *_buffer - RegEx to extract escaped sequences from
            **escs    - Pointer to a array of pointers, to write the escaped
                          character to.

    Outputs: Pointer to the new RegEx with the removed escape sequences.
*******************************************************************************/
BYTE *HandleEscapes(Parser_str *P, BYTE *_buffer, char **escs)
{
  int   spos, esc, i;
  int   write  = 0;
  int   wval   = 1;
  int   escpos = 0;
    /* {"Escape", "Replace with", ...} Preceed Replace with a l for lit, r for*
     *  range, a for assertion, q for quantifer                               */
  const char *escapes[] = {
                       /* C Escape Codes */
                     "\\\\", "l\\", "\\n",  "l\n",
                     "\\r",  "l\r", "\\t",  "l\t",
                     "\\v",  "l\v", "\\f",  "l\f",
                     "\\a",  "l\a",

                       /* Non Greedy Quantifiers */
                     "q",  "lq",
                     "??", "q?", "+?", "q+", "*?", "q*",

                       /* Ranges */
                     "\\d",  "r[0-9]",         "\\D",  "r[^0-9]",
                     "\\w",  "r[A-Za-z0-9_]",  "\\W",  "r[^A-Za-z0-9_]",
                     "\\s",  "r[ \t\r\n\v\f]", "\\S",  "r[^ \t\r\n\v\f]",

                       /* Assertions */
                     "a",    "la",
                     "\\b",  "ab", "\\B", "aB", "\\<", "a<",
                     "\\>",  "a>", "^",   "a^", "$",   "a$",

                       /* Escaped special charcters */
                     "\\.",  "l.", "\\[", "l[", "\\?", "l?",
                     "\\+",  "l+", "\\*", "l*", "\\(", "l(",
                     "\\|",  "l|", "\\)", "l)", "\\^", "l^",
                     "\\$",  "l$"
                    };

  BYTE *buffer = (BYTE *)Dup((char *)_buffer);

  if(!buffer)
  {
    P->Error = PARSER_OUT_OF_MEMORY;
    return 0;
  }

  for(spos = 0; buffer[spos]; ++spos)
  {
    for(esc = 0; esc < (int)(sizeof(escapes) / sizeof(char *)); esc += 2)
    {
      for(i = 0; escapes[esc][i] && buffer[spos + i] == escapes[esc][i];) ++i;
      if(!escapes[esc][i])
      {
        buffer[write++] = (unsigned char)wval;
        escs[escpos++]  = (char *)escapes[esc + 1];
        spos += i - 1;
        if(++wval > 2)
          wval = 1;
        if(escpos >= 510)
        {
          free(buffer);
          return 0;
        }
        break;
      }
    }

    if(esc == sizeof(escapes) / sizeof(char *))
      buffer[write++] = buffer[spos];
  }
  buffer[write] = 0;
  return buffer;
}


/*******************************************************************************
   Function: BuildRange()

Description: Converts a "[...]" sequence into it's binary range representation

     Inputs: *exp    - Pointer to the RegEx to convert to a range
             *_range - Pointer to the range to write the results to
            **escs   - Pointer to the escape sequences to check for
             *esc    - Pointer to the current location in escs

    Outputs: Location in the RegEx to continue parsing from.
*******************************************************************************/
BYTE *BuildRange(BYTE *exp, BYTE *_range, char **escs, int *esc)
{
  BYTE range[32] = {0};
  int  i;
  BYTE c;
  BYTE l   = 0; /* Last charcter */
  int _not = 0;

  c = *++exp;

  if(c == 1 || c == 2)
    if(escs[*esc][0] == 'a' && escs[*esc][1] == '^')
      c = escs[(*esc)++][1];

  switch(c)
  {
    case '^': _not = 1; c = *++exp; if(c != ']') break;
    case ']': SET_BIT_CHAR(range, ']'); l = c; c = *++exp; break;
  }

  for(; c && c != ']'; c = *++exp)
  {
    int min, max;
    min = max = c;
    if(c == 1 || c == 2)
    { /* Extra braces to remove compiler warning */
      if(escs[*esc][0] == 'r')
      {
        ++*esc;
        BuildRange((BYTE *)escs[*esc - 1] + 1, range, escs, esc);
        l = 0;
        continue;
      }
      else
        min = max = escs[(*esc)++][1];
    }

    if(c == '-' && l && exp[1] && exp[1] != ']')
    {
      min = l;
      max = exp[1];
    }

    for(; min <= max; ++min)
      SET_BIT_CHAR(range, min);
    l = (unsigned char)max;
  }

  if(!c)
    --exp;

  if(_not)
    for(i = 0; i < 32; ++i)
      range[i] = (unsigned char)~range[i];

  for(i = 0; i < 32; ++i)
    _range[i] |= range[i];

  _range[0] &= ~(BYTE)0x01;

  return exp;
}

/*******************************************************************************
   Function: Compile()

Description: Compiles a regex into it's byte code form. Once compiled, this
               function will append the original RegEx string to the end of
               the byte code. Memory is layed out as follows:
                   // Byte Code Header
                 int  Len;      // Length/end of the byte code (includes header)
                 int  MThreads; // Max number of threads the RegEx can use

                   // Op Codes.
                 BYTE OpCode;   // Instruction to process
                 ...  Params;   // OpCode paramaters
                 ...

                   // End of the Byte Code
                   // Appended String
                 int  Len;      // Length of the string + NULL Terminator
                 char Str[Len]; // Appened string

     Inputs: *_exp - RegEx to compile.

    Outputs: 0  - Error occured
             1+ - Compiled byte code
*******************************************************************************/
BYTE *Compile(Parser_str *P, char *_exp)
{
  BYTE *exp = (BYTE *)_exp;
  BYTE *exp_base;     /* Pointer to the base of the expression                */
  BYTE  c, w;         /* c - Command, w - Actual character                    */
  BYTE range[32];     /* Used to build a range (IE, [a-z] & similar)          */
  BYTE *data;         /* Compiled byte code                                   */
  int   datasize;     /* Size of data                                         */
  int   i;            /* Loop & general purpose                               */
  int   dpos;         /* Where in data to write to                            */
  int   nlit    = 0;  /* Number of continuous literals                        */
  int   nalt    = 0;  /* Number of alternative/logical or                     */
  int   mthread = 1;  /* Max number of possible threads                       */
  int   stack [512];  /* Stores nlit & nalt when a subexpression is found     */
  int   groups[512];  /* Stores start/end location of groups of characters    */
  char *escs  [512];  /* Stores all the escape sequences found in the _exp    */
  int   stackpos = 2; /* Where in stack to write to                           */
  int   grouppos = 1; /* Where in groups to write to                          */
  int   esc      = 0;

  datasize = 1024;
  data = (BYTE *)malloc(datasize);
  if(!data){P->Error = PARSER_OUT_OF_MEMORY;return 0;}

  exp_base = exp = HandleEscapes(P, exp, escs);

  if(!exp_base)
  {
    free(data);
    return 0;
  }

  groups[0] = 0;

    /* len of data, max threads */
  dpos = 8;

  data[28] = PASS;

  w = c = *exp;
  for(; c; w = c = *++exp)
  {
    if(dpos + 20 >= datasize)
    {
      datasize += 1024;
      data = (BYTE *)realloc(data, datasize);

      if(!data)
      {
        free(exp_base);
        P->Error = PARSER_OUT_OF_MEMORY;
        return 0;
      }
    }

    if(c == 1 || c == 2)
    { /* Extra braces to remove compile warning */
      if(escs[esc][0] == 'l')
        w = escs[esc++][1];
      else
      if(escs[esc][0] == 'r')
        c = '[';
      else
      if(escs[esc][0] == 'q')
      {
        c = escs[esc++][1];
        w = 'q';
      }
      else
      {
        c = 'a';
        w = escs[esc++][1];
      }
    }

    switch(c)
    {
      case '(':
        if(nlit > 1)
        {
          --nlit;
          --grouppos;
        }

        stack[stackpos++] = nlit;
        stack[stackpos++] = nalt;
        nalt = nlit = 0;

        if(stackpos >= 510)
        {
          free(exp_base);
          free(data);
          return 0;
        }
        break;

      case '|':
        while(--nlit > 0)
          --grouppos;

        ++nalt;
        break;

      case ')':
        while(--nlit > 0)
          --grouppos;

        while(nalt-- > 0)
        {
          i = groups[grouppos - 1] + 5;

          dpos = WriteThreadBefore(data, dpos, groups, &grouppos, JUMP, -1);
          --grouppos;
          dpos = WriteThreadBefore(data, dpos, groups, &grouppos, THREAD, i);
        }

        nalt = stack[--stackpos];
        nlit = stack[--stackpos];
        nlit++;

        if(stackpos <= 1)
        {
          free(exp_base);
          free(data);
          return 0;
        }
        break;

      case '?':
        if(c == w)
          dpos = WriteThreadBefore(data, dpos, groups, &grouppos, THREAD, -1);
        else
        {
          i = groups[grouppos - 1];
          dpos = WriteThreadBefore(data, dpos, groups, &grouppos, JUMP, -1);
          dpos = WriteThreadBefore(data, dpos, groups, &grouppos,
                                   THREAD_NON_GREEDY, i + 5);
        }
        break;

      case '*':
        if(c == w)
          data[dpos++] = THREAD;
        else
          data[dpos++] = THREAD_NON_GREEDY;
        SET_INT_INC(data, dpos, groups[grouppos - 1]);
        dpos = WriteThreadBefore(data, dpos, groups, &grouppos, JUMP, dpos - 5);
        break;

      case '+':
        if(c == w)
          data[dpos++] = THREAD;
        else
          data[dpos++] = THREAD_NON_GREEDY;
        SET_INT_INC(data, dpos, groups[grouppos - 1]);
        break;

      case '.':
        if(nlit > 1)
        {
          --nlit;
          --grouppos;
        }
        nlit++;

        groups[grouppos++] = dpos;
        data[dpos++] = WILD;
        ++mthread;
        break;

      case '[':
        if(nlit > 1)
        {
          --nlit;
          --grouppos;
        }
        nlit++;

        groups[grouppos++] = dpos;

        memset(range, 0, 32);
        if(c != w)
        {
          ++esc;
          BuildRange((BYTE *)escs[esc - 1] + 1, range, escs, &esc);
        }
        else
          exp = BuildRange(exp, range, escs, &esc);

        data[dpos++] = RANGE;
        for(i = 0; i < 32; ++i)
          data[dpos++] = range[i];
        ++mthread;
        break;

      case 'a': /* Assertion */
        switch(w)
        {
          case 'b': data[dpos++] = WORD_BOUNDRY;     break;
          case 'B': data[dpos++] = WORD_BOUNDRY_NOT; break;
          case '<': data[dpos++] = WORD_START;       break;
          case '>': data[dpos++] = WORD_END;         break;
          case '^': data[dpos++] = LINE_START;       break;
          case '$': data[dpos++] = LINE_END;         break;
        }
        break;

      default:
        if(nlit > 1)
        {
          --nlit;
          --grouppos;
        }
        nlit++;

        groups[grouppos++] = dpos;
        data[dpos++] = LITERAL;
        data[dpos++] = w;
        ++mthread;
        break;
    }
  }

  while(--nlit > 0)
    --grouppos;

  while(--nalt >= 0)
  {
    i = groups[grouppos - 1] + 5;

    dpos = WriteThreadBefore(data, dpos, groups, &grouppos, JUMP, -1);
    --grouppos;
    dpos = WriteThreadBefore(data, dpos, groups, &grouppos, THREAD, i);
  }

  if(grouppos != 1 && grouppos != 2)
  {
    free(exp_base);
    free(data);
    return 0;
  }

  data[dpos++] = PASS;

  SET_INT(data, 0, dpos);
  SET_INT(data, 4, mthread);

  free(exp_base);

  if(mthread >= P->RegEx->NThreads)
  {
    if(P->RegEx->Threads)
      P->RegEx->Threads = (Thread_str *)realloc(P->RegEx->Threads,
                                         sizeof(Thread_str) * (mthread + 1));
    else
      P->RegEx->Threads = (Thread_str *)malloc(sizeof(Thread_str) *(mthread+1));

    if(!P->RegEx->Threads)
    {
      P->Error = PARSER_OUT_OF_MEMORY;
      free(data);
      return 0;
    }

    P->RegEx->NThreads = mthread + 1;
  }

  if(dpos >= P->RegEx->NProcs)
  {
    if(P->RegEx->Procs)
      P->RegEx->Procs = (int *)realloc(P->RegEx->Procs, sizeof(int) * (dpos+1));
    else
      P->RegEx->Procs = (int *)malloc(sizeof(int) * (dpos + 1));

    if(!P->RegEx->Procs)
    {
      P->Error = PARSER_OUT_OF_MEMORY;
      free(data);
      return 0;
    }

    P->RegEx->NProcs = dpos + 1;
  }

    /* Append the origial expression to the end of the byte code */
  nlit = strlen(_exp);
  if(dpos + nlit + 6 >= datasize)
  {
    datasize += nlit + 6;
    data = (BYTE *)realloc(data, datasize);
    if(!data)
    {
      P->Error = PARSER_OUT_OF_MEMORY;
      return 0;
    }
  }

  SET_INT_INC(data, dpos, nlit);

  for(i = 0; i <= nlit; ++i)
    data[dpos++] = ((BYTE *)_exp)[i];

    /* Resizer the byte code to it's actual size */
  data = (BYTE *)realloc(data, dpos);

  if(!data)
  {
    P->Error = PARSER_OUT_OF_MEMORY;
    return 0;
  }

  /*PrintCompiled(data);*/
  /*printf("\nM Threads: %d\n", mthread); */
  return data;
}

/*******************************************************************************
   Function: InvalidRegEx()

Description: Performs some basic error checks to validate a RegEx. Currently,
               this just checks for unbalanced () and [].

     Inputs: *_exp - RegEx to validate

    Outputs: 0  - No error detected
             1+ - Error code
*******************************************************************************/
int InvalidRegEx(Parser_str *P, const char *_exp)
{
  unsigned int i;
  unsigned int parans = 0;
  unsigned int squarebrackets = 0;
  BYTE *exp = (BYTE *)_exp;
  BYTE *exp_base;     /* Pointer to the base of the expression                */
  char *escs[512];    /* Stores all the escape sequences found in the _exp    */
  int   errors = 0;

  if(!exp)
    return PARSER_TOKEN_NULL;

  exp_base = exp = HandleEscapes(P, exp, escs);

  if(!exp)
    return PARSER_OUT_OF_MEMORY;

  for(i = 0; exp[i]; ++i)
    switch(exp[i])
    {
      case '(':
        ++parans;
        break;

      case ')':
        --parans;
        break;

      case '[':
        ++squarebrackets;
        break;

      case ']':
        --squarebrackets;
        break;
    }

  free(exp_base);

  if(parans)         errors |= PARSER_UNBALANCED_PARENS;
  if(squarebrackets) errors |= PARSER_UNBALANCED_BRACKETS;
  if(errors)         errors |= PARSER_REGEX_COMPILE_ERROR;

  return errors;
}

/*******************************************************************************
   Function: UpdateThreads()

Description: Iteratoes through all the threads, and processes the specifed
               byte code

     Inputs: *buf     - Pointer to a BYTE[3].
                          buf[0] - Last Processed Character
                          buf[1] - Current Character
                          buf[2] - Next Character
             *data    - Pointer to the RegEx byte code
             *threads - Pointer to the thread states
             *nt      - Pointer to the number of threads
             *procs   - Pointer to the proc list
              loop    - Current loop iteration
             *greed   - Pointer to the current greed level


    Outputs: 0 - No RegEx thread hit the PASS op code
             1 - At least one RegEx hit the PASS op code
*******************************************************************************/
int UpdateThreads(Parser_str *P, BYTE *buf, BYTE *data,  Thread_str *threads,
                  int  *nt,  int  *procs, int         loop,    int *greed)
{
  int  go_to;             /* Stores the destination of a JUMP or THREAD       */
  int  run;               /* Stop condition for each thread                   */
  int  t;                 /* Current Thread                                   */
  int  dpos;              /* Location in data                                 */
  int  end      = 0;      /* Was the end found?                               */
  int  nthreads = *nt;    /* Total number of threads                          */
  BYTE c        = buf[1]; /* Cached current character                         */

  for(t = 0; t < nthreads; ++t)
  {
    dpos = threads[t].DPos;
    run  = 1;
    procs[dpos] = 0;

    while(run)
    {
      if(procs[dpos] == loop)
      {
        POP_THREAD
        break;
      }

      procs[dpos]  = loop;

      switch(data[dpos++])
      {
        case PASS:
          if(threads[t].Greed <= *greed)
          {
            end = 1;
            *greed = threads[t].Greed;
          }

          POP_THREAD
          run = 0;
          break;

        case LITERAL:
          if(P->Flags & PARSER_CASE_INSENSITIVE)
          {
            if(TOLOWER(c) != data[dpos] && TOUPPER(c) != data[dpos])
              POP_THREAD
            else
              threads[t].DPos = dpos + 1;
            ++dpos;
          }
          else
            if(c != data[dpos++])
              POP_THREAD
            else
              threads[t].DPos = dpos;

          run = 0;
          break;

        case RANGE:
          if(P->Flags & PARSER_CASE_INSENSITIVE)
          {
            BYTE l = TOLOWER(c);
            BYTE u = TOUPPER(c);
            if(!(data[dpos + (l >> 3)] & (1 << (l & 7))) &&
               !(data[dpos + (u >> 3)] & (1 << (u & 7))))
              POP_THREAD
            else
            {
              dpos += 32;
              threads[t].DPos = dpos;
            }
          }
          else
            if(!(data[dpos + (c >> 3)] & (1 << (c & 7))))
              POP_THREAD
            else
            {
              dpos += 32;
              threads[t].DPos = dpos;
            }
          run = 0;
          break;

        case WILD:
          threads[t].DPos = dpos;
          run = 0;
          break;

        case WORD_BOUNDRY:
          if(IS_WORD(c))
          {
            if(IS_WORD(buf[0]))
            {
              POP_THREAD
              run = 0;
              break;
            }
          }
          else
            if(!IS_WORD(buf[0]))
            {
              POP_THREAD
              run = 0;
            }
          break;

        case WORD_BOUNDRY_NOT:
          if(IS_WORD(c))
          {
            if(!IS_WORD(buf[0]) || !IS_WORD(buf[1]))
            {
              POP_THREAD
              run = 0;
              break;
            }
          }
          else
            if(IS_WORD(buf[0]) || IS_WORD(buf[1]))
            {
              POP_THREAD
              run = 0;
            }
          break;

        case WORD_START:
          if(!IS_WORD(c) || IS_WORD(buf[0]))
          {
            POP_THREAD
            run = 0;
          }
          break;

        case WORD_END:
          if(IS_WORD(c) || !IS_WORD(buf[0]))
          {
            POP_THREAD
            run = 0;
          }
          break;

        case LINE_START:
          if(buf[0] != 0 && buf[0] != '\n' && buf[0] != '\r')
          {
            POP_THREAD
            run = 0;
          }
          break;

        case LINE_END:
          if(c != 0 && c != '\n' && c != '\r')
          {
            POP_THREAD
            run = 0;
          }
          break;

        case JUMP:
          dpos = GET_INT(data, dpos);
          break;

        case THREAD:
          go_to = GET_INT_INC(data, dpos);
          if(procs[go_to] != loop)
          {
            threads[nthreads  ].Greed = threads[t].Greed;
            threads[nthreads++].DPos  = go_to;
            procs[go_to]        = loop;
          }
          break;

        case THREAD_NON_GREEDY:
          go_to = GET_INT_INC(data, dpos);
          if(procs[go_to] != loop && threads[t].Greed < *greed)
          {
            threads[nthreads  ].Greed = threads[t].Greed + 1;
            threads[nthreads++].DPos  = go_to;
            procs[go_to]        = loop;
          }
          break;
      } /* switch(data[dpos++])          */
    } /* while(run)                    */
  } /* for(t = 0; t < nthreads; ++t) */

  *nt = nthreads;

  return end;
}

/*******************************************************************************
   Function: ShiftRight()

Description: RegEx Helper function. Shifts a compiled regex to the right and
               updates any JUMP or THREAD op codes.

     Inputs: *data     - Pointer to the compiled RegEx to update
              start    - Index of the first byte to move.
              end      - Index of the last byte to move. Note: Inclusive
              offset   - How many bytes to shift everything over by
             *groups   - Pointer to the character group list
              grouppos - Current index in groups

    Outputs: New end of the byte code
*******************************************************************************/
int ShiftRight(BYTE *data, int start,int end,int offset,int*groups,int grouppos)
{
  int i;
  int go_to;

    /* Shift everything over by offset */
  for(i = end - 1; i >= start; --i)
    data[i + offset] = data[i];

    /* Fix any THREAD or JUMP  */
  for(i = start + offset; i < end + offset;)
    switch(data[i++])
    {
      case LITERAL: ++i; break;
      case JUMP:
      case THREAD:
      case THREAD_NON_GREEDY:
        go_to = GET_INT(data, i);
        if(go_to >= start)
          SET_INT(data, i, go_to + offset);
        i += 4;
        break;
    }

  for(i = 0; i < grouppos; ++i)
    if(groups[i] >= start)
      groups[i] += offset;

  return end + offset;
}

/*******************************************************************************
   Function: WriteThreadBefore()

Description: Writes a op code (JUMP or THREAD) at the specified location. If
               go_to is set to -1, the last position in the byte code will be
               used.

     Inputs: *data     - Pointer to the byte code to modify
              dpos     - End of the byte code
             *groups   - List of character groups
             *grouppos - Current number of groups
              cmd      - Op code to write
              go_to    - Position to set the THREAD/JUMP to go to

    Outputs: New end location of the byte code
*******************************************************************************/
int WriteThreadBefore(BYTE *data, int dpos, int *groups, int *grouppos,
                      BYTE  cmd,  int go_to)
{
  int start = groups[*grouppos - 1];
  dpos = ShiftRight(data, start, dpos, 5, groups, *grouppos);
  data[start] = cmd;
  groups[*grouppos - 1] = start++;
  if(go_to == -1)
    SET_INT(data, start, dpos)
  else
    SET_INT(data, start, go_to + 5)
  return dpos;
}
